script I wrote to verify RAR files (testrars)

#!/bin/bash

# (C) 2020 Zefie Networks

# This script scans for .rar files in a directory, and runs a test on them using the unrar command
# This script requires the nonfree version of unrar and is only tested with v5

# The results of the scan will be saved in a file (defined by Z_TESTDAT below)

# The file is designed to be both human readable and readable by this script, so do not modify
# the output file except to delete a line to force revalidation (DOS/Unix linefeeds shouldnt matter).

# This script will stop if it reaches a damaged RAR if Z_ABORT_ON_ERROR is defined.

# This script will only validate files that have either not been checked yet, modified since the last check,
# or it has been Z_EXPIRE_TIME since the last validation.


# CONFIG

# How long since the last successful validation before we validate the file again?
Z_EXPIRE_TIME="250:0:0:0" # days:hours:minutes:seconds

# Filename for test metadata, will be placed in directory specified to scan
# Layout is tab-delimited, as follows
# *** Date Validated    Test Status Recovery Record Status  RAR Size    RAW Size    Filename
Z_TESTDAT_FILENAME="rar_test_status.txt"

# You can change these if you want but be aware of tabbing and cosmetic issues
Z_OKSTR="PASSED (OK)"
Z_RECOVERY_RECORD_OKSTR="Has Recovery Record"
Z_RECOVERY_RECORD_ERRSTR="NO RECOVERY RECORD"
Z_DATESTR="%Y-%m-%d %H:%M:%S"
Z_DATESTR_DAY="%A, %B %d %Y"

# Comment this out to continue scanning even if a rar fails to validate
Z_ABORT_ON_ERROR=1
# Uncomment this out to fail if a rar does not have a recovery record
#Z_ABORT_ON_NRR=1

# END EZ CONFIG





Z_SCRIPTDIR="$(realpath "$(dirname "${0}")")"
Z_ERRCNT=0
Z_ERRRCNT=0
Z_NRRCNT=0
Z_MISSING=0

# shellcheck source=/home/zefie/bin/ansihelper.sh
source "${Z_SCRIPTDIR}/ansihelper.sh"

if [ -z "${1}" ]; then
    # script requires an arg to run
    echo "Usage: ${0} <directory> [directory ...]"
    exit 1;
fi

if [ "${1}" == "-nf" ]; then
    unset Z_ABORT_ON_ERROR;
    shift;
fi

for arg in "${@}"; do
    if [ ! -d "${arg}" ]; then
        echo "ERR: Not a directory: \"${1}\""
    fi
done

# Intercept ^C and finish up cleanly instead of getting stuck in the loop
trap ctrl_c INT
function ctrl_c() {
        echo "** ^C pressed, aborting script, please wait..."
    export Z_ABORT=1;
}

function z_chk_if_single_or_part1_rar() {
    unset Z_MULTIPART_RAR
    # Check if a single part rar, or if part1 of a multipart, returns 1 if not.
    if [ "$(echo "${1}" | grep -Eic '\.part[0-9]{0,3}\.rar$')" -gt 0 ]; then
        if [ "$(echo "${1}" | grep -Eic '\.part[0]{0,2}1\.rar$')" -eq 0 ]; then
            return 1;
        fi
        export Z_MULTIPART_RAR=1
    fi
}

function bytesToHuman() {
    # Simple 'friendlyBytes' from stackoverflow
    b=${1:-0}; d=''; s=0; S=(Bytes {K,M,G,T,P,E,Z,Y}iB)
    while ((b > 1024)); do
        d="$(printf ".%02d" $((b % 1024 * 100 / 1024)))"
        b=$((b / 1024))
        (( s++ ))
    done
    echo "$b$d ${S[$s]}"
}

function z_strip_empty_array() {
    # Usage: z_strip_empty_array $arrayname $thearray
    # (Re)defines the array of $arrayname with the contents of $thearray, but without null entries (and re-sorted)
    local Z_ARRY="${1}"
    shift
    eval "declare -a ${Z_ARRY}"
    for i in "${@}"; do
       if [ -z "$i" ]; then
         continue
       fi
       eval "${Z_ARRY}+=(\"${i}\")"
    done
}

function z_strip_from_txt() {
    # Rebuilds text file $2 without the lines containing $1
    local Z_TMP
    Z_TMP="$(mktemp)"
    grep -Fv "${1}" "${2}" > "${Z_TMP}"
    cat "${Z_TMP}" > "${2}"
    rm "${Z_TMP}"
}

function z_fileheader() {
    # Rebuilds the text header (starting with ***) without destroying the file contents
        # No args, replace $Z_TESTDAT with $1 to make into a arg type function
    # You can add extra tabs for formatting (eg \t\t) without breaking the script, thanks to z_strip_empty_array
    local Z_TMP Z_EXPIRE
    Z_TMP="$(mktemp)"
    Z_EXPIRE=$(("$(date -u +%s)" + "$(echo -n "${Z_EXPIRE_TIME}" | awk -F: '{ print ($1 * 86400) + ($2 * 3600) + ($3 * 60) + $4 }')"))

    if [ -f "${1}" ]; then
        grep -Fv "***" "${1}" > "${Z_TMP}"
    fi
    echo "*** RAR Integrty Test Status for ${Z_DIR} (Times in UTC/GMT) ***" > "${1}"
    {
        echo "*** Last Updated: $(date -u "+${Z_DATESTR}") (Suggested Revalidation: $(date -u --date="@${Z_EXPIRE}" "+${Z_DATESTR_DAY}")) ***"
        echo -e '*** Date Validated\tTest Status\tRecovery Record Status\tRAR Size\tRAW Size\tFile Last Modified\tFilename'
        cat "${Z_TMP}"
    } >> "${1}"
    rm "${Z_TMP}"
}

function z_chk_abort() {
    # If the abort flag has been set prior, exit now
    if [ ! -z "${Z_ABORT}" ]; then
        exit "${Z_ABORT}";
    fi
}

function do_rar_test() {
    z_chk_abort
    local Z_STATUS="UNKNOWN ERR"
    local Z_STATUS_C="yellow"
    local Z_REC_C="yellow"
    local Z_RRES Z_TMP Z_REC Z_COMPSIZE Z_RAWSIZE Z_LASTMOD Z_REC_REC Z_REC_RES;

    # Check for recovery record
    Z_REC_REC="$(unrar l "${1}" | grep "recovery record" -c)"
    Z_REC_RES=$?
    if [ ${Z_REC_RES} -ne 0 ] || [ "${Z_REC_REC}" -lt 1 ]; then
        Z_REC="${Z_RECOVERY_RECORD_ERRSTR}"
        Z_NRRCNT=$((Z_ERRCNT + 1))
        if [ ! -z "${Z_ABORT_ON_NRR}" ]; then
            Z_ABORT=1
        fi
        Z_REC_C="red"
    else
        Z_REC="${Z_RECOVERY_RECORD_OKSTR}"
        Z_REC_C="green"
    fi

    # Test with unrar (7z/7za/p7zip will return errcode 0 on corrupt file and cannot be used)
    unrar t "${1}"
    Z_RRES=$?
    if [ ${Z_RRES} -eq 0 ]; then
        Z_STATUS="${Z_OKSTR}"
        Z_STATUS_C="green"
    else
        case ${Z_RRES} in
            255)
                # When user hits ^C
                Z_STATUS="FAILED (ABORT)"
                Z_ABORT=1
                ;;
            10)
                Z_STATUS="FAILED (NOTRAR)"
                ;;

            3)
                if [ "${Z_REC}" == "${Z_RECOVERY_RECORD_OKSTR}" ]; then
                    # corrupt and has recovery record
                    Z_STATUS="NEEDS REPAIR"
                    Z_ERRRCNT=$((Z_ERRRCNT))
                else
                    # corrupt and does NOT have recovery record
                    Z_STATUS="FAILED (CORRUPT)"
                fi
                ;;
            *)
                # catchall for unknown error codes
                Z_STATUS="FAILED (${Z_RRES})"
                ;;
        esac
        if [ ! -z "${Z_ABORT_ON_ERROR}" ]; then
            Z_ABORT=1
        fi
        Z_STATUS_C="red"
        Z_ERRCNT=$((Z_ERRCNT + 1))
    fi
    z_strip_from_txt "${1}" "${2}"
    z_fileheader "${2}"

        if [ -z "${Z_MULTIPART_RAR}" ]; then
                # Use stat to get filesize of a single file
                Z_COMPSIZE="$(bytesToHuman "$(stat -c %s "${1}")")"
        else
                # Use stat to get the size of all the rar parts, and add them together
        Z_MULTIRAR_PREFIX="$(echo "${1}" | rev | cut -d'.' -f3- | rev).part"
        Z_COMPSIZE=0
        for fss in $(stat -c %s "${Z_MULTIRAR_PREFIX}"*); do
            Z_COMPSIZE=$(( Z_COMPSIZE + fss ))
        done
        Z_COMPSIZE=$(bytesToHuman ${Z_COMPSIZE})
        fi

    Z_RAWSIZE="$(bytesToHuman "$(unrar l "${1}" | tail -n2 | head -n1 | awk ' { print $1 } ')")"
    Z_LASTMOD="$(date -u "+${Z_DATESTR}" --date="@$(stat -c "%Y" "${1}")")"
    echo -e "$(date -u "+${Z_DATESTR}")\\t${Z_STATUS}\\t${Z_REC}\\t${Z_COMPSIZE}\\t${Z_RAWSIZE}\\t${Z_LASTMOD}\\t${1}" >> "${2}"
    echo -e "$(str_color "${Z_STATUS_C}" "${Z_STATUS}")\\t$(str_color "${Z_REC_C}" "${Z_REC}")\\t${1}"
}

function z_process_directory() {
    local Z_DIR Z_TESTDAT Z_TS Z_EXPIRE Z_LASTMOD Z_RES Z_PWD Z_MKTEMP;

    Z_DIR="$(realpath "${1}")"
    Z_PWD="${PWD}"
    Z_TESTDAT="${Z_DIR}/${Z_TESTDAT_FILENAME}"
    cd "${Z_DIR}" || (echo "Could not enter directory: ${Z_DIR}" && exit 1)

    while IFS=$'\n' read -r -d '' file
    do
        z_chk_abort
        z_chk_if_single_or_part1_rar "${file}" || continue;

        # if the test result file doesn't exist
        if [ ! -f "${Z_TESTDAT}" ]; then
            # then create it and insert the header
            touch "${Z_TESTDAT}" || exit 1;
            z_fileheader "${Z_TESTDAT}"
        fi

        # scan the test result file for this filename
        IFS=$'\t' read -r -a Z_RES <<< "$(grep -F "${file}" "${Z_TESTDAT}")"
        z_strip_empty_array Z_RES "${Z_RES[@]}";

        # if there was a match from the test result file
        if [ ${#Z_RES[@]} -gt 2 ]; then
            # get the stored information and ch eck it
            Z_TS="$(date -u --date="${Z_RES[0]}" +%s 2>/dev/null || echo -n "${Z_RES[0]}")"
            Z_EXPIRE=$(("${Z_TS}" + "$(echo -n "${Z_EXPIRE_TIME}" | awk -F: '{ print ($1 * 86400) + ($2 * 3600) + ($3 * 60) + $4 }')"))
            Z_LASTMOD="$(date -u "+${Z_DATESTR}" --date="@$(stat -c "%Y" "${file}")")"

            # if the expire time elapsed,                  the last test wasn't ok,            or the modified time changed
            if [ "$(date -u +%s)" -gt "${Z_EXPIRE}" ] || [ "${Z_RES[1]}" != "${Z_OKSTR}" ] || [ "${Z_LASTMOD}" != "${Z_RES[5]}" ]; then
                # then validate the file
                do_rar_test "${file}" "${Z_TESTDAT}"
            else
                # scan status file for any rars that don't have recovery records and count them
                if [ "${Z_RES[2]}" != "${Z_RECOVERY_RECORD_OKSTR}" ]; then
                    Z_NRRCNT=$((Z_NRRCNT + 1))
                fi
            fi
        else
            # otherwise validate the file
            do_rar_test "${file}" "${Z_TESTDAT}"
        fi
    done <   <(find . -name '*.rar' -print0);

    Z_MKTEMP=$(mktemp)
    while IFS=$'\n' read -r line
    do
        z_chk_abort
        if [ "$(echo "${line}" | grep -F "***" -c)" -gt 0 ]; then
            echo "${line}" >> "${Z_MKTEMP}"
            continue;
        fi
        IFS=$'\t' read -r -a Z_TRES <<< "${line}"
        if [ ! -z "${Z_TRES[6]}" ]; then
            if [ ! -f "${Z_TRES[6]}" ]; then
                Z_MISSING=$((Z_MISSING + 1))
            else
                echo "${line}" >> "${Z_MKTEMP}"
            fi
        fi
    done < "${Z_TESTDAT}";

    if [ "${Z_MISSING}" -gt 0 ]; then
        cat "${Z_MKTEMP}" > "${Z_TESTDAT}"
    fi
    rm "${Z_MKTEMP}"

    cd "${Z_PWD}" || (echo "Could not return to original directory: ${Z_PWD}" && exit 1)
    if [ "${Z_ERRCNT}" -eq 0 ] && [ "${Z_NRRCNT}" -eq 0 ]; then
        # all good
        echo "All $(str_color green "${Z_OKSTR}") for ${Z_DIR}"
    else
        echo "$(str_color red "Error report") for ${Z_DIR}:"
        local Z_ERRRCNT_STR;
        if [ "${Z_ERRRCNT}" -eq "${Z_ERRCNT}" ]; then
            Z_ERRRCNT_STR="$(str_color green all)";
        elif [ "${Z_ERRRCNT}" -gt 0 ]; then
            Z_ERRRCNT_STR="$(str_color yellow "${Z_ERRRCNT}")"
        else
            Z_ERRRCNT_STR="$(str_color red none)"
        fi
        if [ "${Z_ERRCNT}" -gt 0 ]; then
            echo "$(str_color red "${Z_ERRCNT} file(s) failed") to validate (of those, ${Z_ERRRCNT_STR} of them can be repaired)."
        fi
        if [ "${Z_NRRCNT}" -gt 0 ]; then
            echo "$(str_color yellow "${Z_NRRCNT} file(s)") do not have recovery records."
        fi
    fi
    if [ "${Z_MISSING}" -gt 0 ]; then
        echo "$(str_color yellow "Removed ${Z_MISSING} missing file(s)") from $(str_color cyan "${Z_TESTDAT}")"
    fi
}

for dir in "${@}"; do
    z_chk_abort
    z_process_directory "${dir}"
done