lawn-hark: Parallelization using xargs(1). - gopher-lawn - The gopher lawn gopher directory project.
 (HTM) git clone git://bitreich.org/gopher-lawn/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/gopher-lawn/
 (DIR) Log
 (DIR) Files
 (DIR) Refs
 (DIR) Tags
       ---
 (DIR) commit e3cc5279072ddbee7f2d58152732c759a9ede582
 (DIR) parent b9a49018389fa0126149e3cbef3ecdef4defe6ef
 (HTM) Author: Julian Schweinsberg <pazz0@0xfa.de>
       Date:   Tue, 19 Dec 2023 20:25:02 +0100
       
       lawn-hark: Parallelization using xargs(1).
       
       Signed-off-by: Christoph Lohmann <20h@r-36.net>
       
       Diffstat:
         M lawn-hark/lawn-hark.sh              |      73 +++++++++++++++++--------------
       
       1 file changed, 41 insertions(+), 32 deletions(-)
       ---
 (DIR) diff --git a/lawn-hark/lawn-hark.sh b/lawn-hark/lawn-hark.sh
       @@ -5,6 +5,7 @@
        
        errorthreshold=3
        timeout=15
       +maxworkers=8
        onionsocksproxy="127.0.0.1:9050"
        
        function tcpdial() {
       @@ -95,26 +96,14 @@ function checkraw() {
                return 0
        }
        
       -checktime="$(date +%s)" 
       -statedir="$1"
       +program="$(readlink -f "$0")"
        
       -if [ -z "${statedir}" ];
       +if [ "${LAWNHARK_WORKER}" = "1" ];
        then
       -        printf "You need to specify a state dir.\n" >&2
       -        exit 1
       -fi
       -
       -mkdir -p "${statedir}"
       -if [ ! -d "${statedir}" ];
       -then
       -        printf "%s is not a directory! Aborting.\n" "${statedir}" >&2
       -        exit 1
       -fi
       +        statedir="$1"
       +        checktime="$2"
       +        f="$3"
        
       -shift
       -
       -for f;
       -do
                type=""
                selector=""
                host=""
       @@ -145,12 +134,10 @@ do
                        || [ -z "${host}" ] \
                        || [ -z "${port}" ];
                then
       -                printf "ERROR\t%s\tInvalid entry!\n" "${f}" >&2
       -                continue
       +                flock -x "${program}" printf "ERROR\t%s\tInvalid entry!\n" "${f}" >&2
       +                exit
                fi
        
       -        #printf "DEBUG\t%s\tchecking\t%s\t%s\t%s\t%s\n" "${f}" "${type}" "${selector}" "${host}" "${port}"
       -
                case "${type}" in
                cso )
                        error="$(checkcso "${host}" "${port}")"
       @@ -182,8 +169,8 @@ do
                                        error="$(checkraw "${sshhost}" "${sshport}")"
                                        ;;
                                * )
       -                                printf "TODO\t%s\tCan't handle %s\n" "${f}" "${url}"
       -                                continue
       +                                flock -x "${program}" printf "TODO\t%s\tCan't handle %s\n" "${f}" "${url}" >&2
       +                                exit
                                        ;;
                                esac
                        else
       @@ -215,20 +202,42 @@ do
        
                if [ ${errorcount} -ge ${errorthreshold} ];
                then
       -                printf "ERROR\t%s\t%s\n" "${f}" "${error}" >&2
       +                flock -x "${program}" printf "ERROR\t%s\t%s\n" "${f}" "${error}" >&2
                fi
        
                printf "%s\t%s\n" "${checktime}" "${errorcount}" > "${statefile}"
       -done
       +else
       +        checktime="$(date +%s)" 
       +        statedir="$1"
        
       -# garbage collection
       -find "${statedir}" -type f | while read -r f;
       -do
       -        IFS="        " read -r lastcheck errorcount < "${f}"
       +        if [ -z "${statedir}" ];
       +        then
       +                printf "You need to specify a state dir.\n" >&2
       +                exit 1
       +        fi
        
       -        if [ ${lastcheck} -ne ${checktime} ];
       +        mkdir -p "${statedir}"
       +        if [ ! -d "${statedir}" ];
                then
       -                rm -f "${f}"
       +                printf "%s is not a directory! Aborting.\n" "${statedir}" >&2
       +                exit 1
                fi
       -done
        
       +        shift
       +
       +        for f;
       +        do
       +                printf "%s\0" "${f}"
       +        done | LAWNHARK_WORKER=1 xargs -r -0 -P "${maxworkers}" -L1 "${program}" "${statedir}" "${checktime}"
       +
       +        # garbage collection
       +        find "${statedir}" -type f | while read -r f;
       +        do
       +                IFS="        " read -r lastcheck errorcount < "${f}"
       +
       +                if [ ${lastcheck} -ne ${checktime} ];
       +                then
       +                        rm -f "${f}"
       +                fi
       +        done
       +fi