debmirror

#!/bin/bash
mirror_root=/var/www/debmirror
exclude_regex="(\?|installer|udeb|changelog|\.(changes|diff)|-(updates|backports)|by-hash/(md5sum|sha1)|arm(64|el|hf)|mips(64|64el|el)?|powerpc|ppc64el|s390x)"

if [ $# -eq 0 ]; then while read name url; do [ -z "${name}" ] && continue; [ -z "${url}" ] && continue; "$0" "${name}" "${url}"; done <<-EOF

    kali    http://http.kali.org/kali
    debian  http://ftp.debian.org/debian

EOF
exit 0; fi

if [ $# -eq 2 ]; then screen -d -m -S "debmirror-$1" "$0" "$1" "$2" "!"; exit 0; fi

do_mirror() {
    [ -z "$1" ] && return; [ -z "$2" ] && return

    local repo_root="${mirror_root}/$1"
    if ! readlink -m "${repo_root}/" | egrep -qe "^${mirror_root}/"; then
        exit 1
    fi

    exclude_regex=$(echo "${exclude_regex}" | sed -re "s#([a-zA-Z])#\[\u\1\l\1\]#g")

    local tmp_dir=$(mktemp -d --tmpdir="${mirror_root}")
    cd "${tmp_dir}/"

    ### clone mirror in temporary directory
    printf "name: %s\n" "$1" >> stats
    printf "uri: %s\n" "$2" >> stats

    local repo_ts=$(date -u -R)
    printf "timestamp: %s\n" "${repo_ts}" >> stats

    local wget_depth=$(echo "${2}" | sed -re "s#/+\$##g;s#^[^:]+:/*[^/]+(/.*)?\$#\1#" | tr -cd '/' | wc -c)

    local wget_start=$(date "+%s")
    wget --quiet --inet4-only --prefer-family=IPv4 --recursive --no-host-directories --no-parent --level=10 --cut-dirs=${wget_depth} --regex-type pcre --reject-regex "${exclude_regex}" "$2/dists/"
    local wget_end=$(date "+%s")
    sleep 1

    local wget_seconds=$[wget_end - wget_start]
    local wget_time=$(date -u -d "@${wget_seconds}" "+%H:%M:%S")
    printf "wget time: %s (%s seconds)\n" "${wget_time}" ${wget_seconds} >> stats

    ### cleanup and re-create destination folder
    ### move files from temporary folder to destination folder
    ### remove temporary folder
    if [ -e "${repo_root}" ] ; then
        rm -rf "${repo_root}/"
    fi
    mkdir -p "${repo_root}/"
    mv * "${repo_root}/"
    cd "${repo_root}/"
    rm -rf "${tmp_dir}/"

    ### force hardlinks in repo
    local repo_size_before=$(du -d0 "${repo_root}/" | egrep -oe "^[0-9]+")
    printf "mirror size (as is): %s\n" ${repo_size_before} >> stats

    local t0=$(mktemp); local t1=$(mktemp)
    local merge_start=$(date "+%s")
    find "${repo_root}/" -mindepth 1 -type f -exec sha256sum -b {} + | sed -re "s#^([0-9a-fA-F]+) +\*?([^ ].*)\$#\1 \2#" > "${t0}"
    cut -d " " -f1 < "${t0}" | sort | uniq > "${t1}"
    while read hsum; do
        local src_file=""
        egrep -e "^${hsum} " "${t0}" | while read tsum tfile; do
            if [ -z "${src_file}" ]; then src_file=${tfile}; continue; fi
            rm "${tfile}"; ln "${src_file}" "${tfile}"
        done
    done < "${t1}"
    local merge_end=$(date "+%s")

    rm "${t0}" "${t1}"; sleep 1

    local repo_size_after=$(du -d0 "${repo_root}/" | egrep -oe "^[0-9]+")
    printf "mirror size (merged): %s\n" ${repo_size_after} >> stats

    local merge_seconds=$[merge_end - merge_start]
    local merge_time=$(date -u -d "@${merge_seconds}" "+%H:%M:%S")
    printf "merge time: %s (%s seconds)\n" "${merge_time}" ${merge_seconds} >> stats

    ### reset timestamps in destination folder
    find "${repo_root}/" -exec touch {} +
}

do_mirror "$@"
exit 0