unpackng.sh

#!/bin/sh
# -*- tab-width:  4 -*- ;; Emacs
# vi: set tabstop=4     :: Vi/ViM
############################################################ INFORMATION
#
# Command Usage:
#
#   ./unpack.sh [-afho] package-archive [package-archive ...]
#
#   OPTIONS:
#       -a     Move the package-archive to an `archive' sub-directory (next
#              to the unpacked `src' directory) after successfully unpacking
#              the package. Not performed by default.
#       -f     Force. Allow unpacking to a directory that already exists.
#       -h     Print this message to stderr and exit.
#       -o     Unpack archive to origin sub-directory. Default is to unpack
#              to the local directory.
#
############################################################ GLOBALS

#
# Global exit status variables
#
SUCCESS=0
FAILURE=1

#
# Program name and directory
#
progname="${0##*/}"
progdir="${0%/*}"

#
# Options
#
FORCE=          # -f
MOVE_PACKAGE=       # -a
USE_ORIGIN_SUBDIR=  # -o

#
# Miscellaneous
#
VALID_VARNAME_CHARS="0-9ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_"
export VALID_VARNAME_CHARS # NB: exported for awk(1) `ENVIRON[]' visibility


# fprintf $fd $fmt [ $opts ... ]
#
# Like printf, except allows you to print to a specific file-descriptor. Useful
# for printing to stderr (fd=2) or some other known file-descriptor.
#
fprintf()
{
    local fd=$1
    [ $# -gt 1 ] || return $FAILURE
    shift 1
    printf "$@" >&$fd
}

# eprintf $fmt [ $opts ... ]
#
# Print a message to stderr (fd=2).
#
eprintf()
{
    fprintf 2 "$@"
}

# die [ $fmt [ $opts ... ]]
#
# Optionally print a message to stderr before exiting with failure status.
#
die()
{
    local fmt="$1"
    [ $# -gt 0 ] && shift 1
    [  "$fmt"  ] && eprintf "$fmt\n" "$@"

    exit $FAILURE
}

# usage
#
# Prints a short syntax statement and exits.
#
usage()
{
    local optfmt="\t%-7s%s\n"

    eprintf "Usage: %s [-afho] package-archive [package-archive ...]\n" \
            "$progname"

    eprintf "OPTIONS:\n"
    eprintf "$optfmt" "-a" \
            "Move the package-archive to an \`archive' sub-directory (next"
    eprintf "$optfmt" "" \
            "to the unpacked \`src' directory) after successfully unpacking"
    eprintf "$optfmt" "" \
            "the package. Not performed by default."
    eprintf "$optfmt" "-f" \
            "Force. Allow unpacking to a directory that already exists."
    eprintf "$optfmt" "-h" \
            "Print this message to stderr and exit."
    eprintf "$optfmt" "-o" \
            "Unpack archive to origin sub-directory. Default is to unpack"
    eprintf "$optfmt" "" \
            "to the local directory."

    die
}

# read_pkg_origin
# Usage:
#   read_pkg_origin DATA [DATA] ...
#       OR
#   read_pkg_origin < FILE
#
# DESCRIPTION:
#   Takes the contents of a packing-list (+CONTENTS) either in the form of data
#   being piped-in, or as one or more arguments provided to this function. When
#   provided as an argument, you must encapsulate the data from each packing-
#   list with double-quotes to preserve the multi-line nature of the data. For
#   example:
#
#   Simple:
#       read_pkg_origin "$(cat ./+CONTENTS)"
#   - or -
#       read_pkg_origin "`cat ./+CONTENTS`"
#
#   Compound:
#       data="$(cat ./+CONTENTS)"
#   - or -
#       data="`cat ./+CONTENTS`"
#   - or -
#       data=$(cat ./+CONTENTS)
#   - or -
#       data=`cat ./+CONTENTS`
#   - then -
#       read_pkg_origin "$data"
#
#   While you should avoid the following [unsupported] syntaxes:
#
#   To be avoided:
#       read_pkg_origin $(cat ./+CONTENTS)
#   - or -
#       read_pkg_origin `cat ./+CONTENTS`
#   - or -
#       read_pkg_origin $data
#
#   Using double-quote encapsulation, you can optionally query multiple
#   packing-lists with the following example syntax:
#
#   Multiple arguments:
#       read_pkg_origin "$(cat pkgA/+CONTENTS)" "$(cat pkgB/+CONTENTS)"
#   - or -
#       dataA=$(cat pkgA/+CONTENTS)
#       dataB=$(cat pkgB/+CONTENTS)
#       read_pkg_origin "$dataA" "$dataB"
#
#   Finally, you can pipe the packing-list data directly into the function. See
#   below:
#
#   Handling PIPE data:
#       cat ./+CONTENTS | read_pkg_origin
#   - or -
#       read_pkg_origin < ./+CONTENTS
#
#   Only the first such ORIGIN field from any/all supplied packing-list data is
#   printed, with the exception of when multiple arguments are provided (when
#   the first ORIGIN field is printed for each argument, as-if each argument
#   was a full-and-complete packing-list from an independent package).
#
read_pkg_origin()
{
    local ORIGIN="@comment ORIGIN:*"

    if [ $# -gt 0 ]; then
        # Parse each argument as a full-and-complete packing-list
        while [ $# -gt 0 ]; do
            echo "$1" | (
            while read LINE; do
                case "$LINE" in $ORIGIN)
                    echo "${LINE#*:}" && break
                esac
            done )
            shift 1
        done
        return $SUCCESS
    fi

    #
    # Expect PIPE data to be provided from STDIN (otherwise, as
    # expected, the user will be prompted to provide the input on
    # the command-line when `read' is invoked
    #
    (
        while read LINE; do
            case "$LINE" in $ORIGIN)
                echo "${LINE#*:}" && break
            esac
        done
    )
}

# manifest_read [-p $prefix] [-r $filter_object] [-R $filter_value] [$file]
# manifest_read [-p $prefix] [-r $filter_object] [-R $filter_value] -i "$data"
#
# Read MANIFEST $file in JSON format. If no other arguments are given, all
# objects/properties are read into the current sh(1) namespace as a series of
# environment variables.  If `-i' is given, take first argument as a full-and-
# complete MANIFEST. If neither $file nor $data is given, read from stdin.
#
# If given `-p $prefix', each environment variable begins with `${prefix}_'.
# If given `-r $filter_object', only objects matching $filter_object regular
# expression are read into the current namespace.
# If given `-R $filter_value', only properties whose value match $filter_value
# regular expression are read into the current namespace.
#
manifest_read_awk='
    BEGIN {
        nkeys[depth = 0] = 0
        building_array = building_string = 0
        valid_chars = ENVIRON["VALID_VARNAME_CHARS"]
        if (filter_object) filter_object = "^" filter_object "$"
        if (filter_value) filter_value = "^" filter_value "$"
    }
    ############################################### FUNCTIONS
    function trim_match()
    {
        match_text = substr($0, RSTART, RLENGTH)
        $0 = substr($0, RSTART + RLENGTH)
        return match_text
    }
    function trim_keyword()
    {
        if (!(keylen = length(key[depth]))) return

        keyword = substr(keyword, 0, length(keyword) - keylen - 1)
        # NB: The "-1" is for keyword separator "_"

        key[depth] = ""
        # NB: Prevents extra calls from reacting (depth retained)
    }
    function objsafe(name)
    {
        gsub("[^" valid_chars "]", "_", name)
        return name
    }
    function json_print(object, value)
    {
        gsub(/'\''/, "'\''\\'\'\''", value)
        object = objsafe(object)
        print object "='\''" value "'\''"
        if (!building_array) print object "_type=scalar"
    }
    function json_filtered_print(object, value)
    {
        if (object !~ filter_object) return
        if (value !~ filter_value) return
        return json_print(object, value)
    }
    function json_unset_value(object)
    {
        print "unset", objsafe(object) "_value"
    }
    function json_filtered_unset_value(object)
    {
        if (object !~ filter_object) return
        return json_unset_value(object)
    }
    function json_objname()
    {
        if (building_array)
            return keyword "_" building_array++
        else if (depth <= 1)
            return keyword

        if ((keylen = length(key[depth]))) keylen++
        # NB: If non-NULL current-depth key, increment for separator

        return substr(keyword, 0, length(keyword) - keylen) \
            "_" nkeys[depth]
    }
    ############################################### MAIN LOOP
    { while ($0) { # Loop until done processing everything on this line
        if (building_string) {
            while (match($0, /^[^"]*\\"/))
                value = value trim_match()
            if (!match($0, /^[^"]*"/)) { # No ending quote
                value = value $0
                next # Continue reading on next line
            }
            building_string = 0
            value = value substr($0, RSTART, RLENGTH - 1)
            trim_match()
            sub(/^[[:space:]]*,[[:space:]]*/, "")
            json_filtered_print(keyword, value)
            trim_keyword()
        }
        ################################### OPENING PATTERNS
        else if (match($0, /^[[:space:]]*{[[:space:]]*/)) {
            nkeys[++depth] = 0
            trim_match()
        } else if (keyword && match($0, /^[[:space:]]*\[/)) {
            building_array = 1
            trim_match()
        }
        ################################### OBJECTS
        else if (match($0, \
            /^[[:space:]]*"[^"]+"[[:space:]]*:[[:space:]]*/ \
        )) {
            nkeys[depth]++
            key[depth] = trim_match()
            sub(/^[[:space:]]*"/, "", key[depth])
            sub(/"[[:space:]]*:[[:space:]]*$/, "", key[depth])
            if (keyword) json_filtered_print( \
                keyword "_" nkeys[depth], objsafe(key[depth]) \
            )
            keyword = keyword ( keyword ? "_" : "" ) key[depth]
        }
        ################################### PROPERTIES
        else if (keyword && match($0, /^[[:space:]]*"/)) {
            value = ""
            trim_match()
            while (match($0, /^[^"]*\\"/))
                value = value trim_match()
            if (!match($0, /^[^"]*"/)) {
                building_string = 1
                value = $0
                next
            }
            value = value substr($0, RSTART, RLENGTH - 1)
            trim_match()
            sub(/^[[:space:]]*,[[:space:]]*/, "")

            object = json_objname()
            if (depth <= 1)
                json_filtered_print(object, value)
            else {
                json_filtered_print(object, key[depth])
                json_filtered_print(object "_value", value)
            }
            if (building_array)
                json_filtered_unset_value(object)
            else
                trim_keyword()
        }
        else if (keyword && match($0, \
            /^[[:space:]]*[^[:space:],}\]]+[[:space:]]*/ \
        )) {
            value = trim_match()
            sub(/^[[:space:]]*/, "", value)
            sub(/[[:space:]]*$/, "", value)
            sub(/^[[:space:]]*,[[:space:]]*/, "")
            json_filtered_print(keyword, value)
            trim_keyword()
        }
        ################################### CLOSING PATTERNS
        else if (match($0, /^[[:space:]]*\][[:space:]]*/)) {
            json_filtered_print(keyword, --building_array)
            json_filtered_print(keyword "_type", "array")
            building_array = 0
            trim_keyword()
            trim_match()
            sub(/^[[:space:]]*,[[:space:]]*/, "")
        }
        else if (match($0, /^[[:space:]]*}[[:space:]]*/)) {
            if (keyword) json_filtered_print(keyword, nkeys[depth])
            building_array = 0 # Pedantic
            nkeys[depth] = 0 # Pedantic
            depth-- # NB: Done prior to calling trim_keyword()
            trim_keyword()
            trim_match()
            sub(/^[[:space:]]*,[[:space:]]*/, "")
        }
    } }
' # END-QUOTE
manifest_read()
{
    local __prefix __obj_regex __value_regex __arg_is_data=

    local OPTIND=1 OPTARG __flag
    while getopts ip:r:R: __flag; do
        case "$__flag" in
        i) __arg_is_data=1 ;;
        p) __prefix="$OPTARG" ;;
        r) __obj_regex="$OPTARG" ;;
        R) __value_regex="$OPTARG" ;;
        \?|*) return $FAILURE
        esac
    done
    shift $(( $OPTIND - 1 ))

    if [ "$__arg_is_data" ]; then
# XXX       eval "$( echo "$1" | awk -v keyword="$__prefix" \
        echo "$( echo "$1" | awk -v keyword="$__prefix" \
            -v filter_object="$__obj_regex" \
            -v filter_value="$__value_regex" \
            "$manifest_read_awk" )"
    else
# XXX       eval "$( awk -v keyword="$__prefix" \
        echo "$( awk -v keyword="$__prefix" \
            -v filter_object="$__obj_regex" \
            -v filter_value="$__value_regex" \
            "$manifest_read_awk" "$@" )"
    fi
}
manifest_read "$@"
exit

# read_pkgng_origin
# Usage:
#   read_pkgng_origin DATA [DATA] ...
#       OR
#   read_pkgng_origin < FILE
#
# DESCRIPTION:
#   Takes the contents of a packing-list (+MANIFEST or +COMPACT_MANIFEST)
#   either in the form of data being piped-in, or as one or more arguments
#   provided to this function. When provided as an argument, you must
#   encapsulate the data from each packing-list with double-quotes to preserve
#   the multi-line nature of the data. For example:
#
#   Simple:
#       read_pkgng_origin "$(cat ./+COMPACT_MANIFEST)"
#   - or -
#       read_pkgng_origin "`cat ./+COMPACT_MANIFEST`"
#
#   Compound:
#       data="$(cat ./+COMPACT_MANIFEST)"
#   - or -
#       data="`cat ./+COMPACT_MANIFEST`"
#   - or -
#       data=$(cat ./+COMPACT_MANIFEST)
#   - or -
#       data=`cat ./+COMPACT_MANIFEST`
#   - then -
#       read_pkgng_origin "$data"
#
#   While you should avoid the following [unsupported] syntaxes:
#
#   To be avoided:
#       read_pkgng_origin $(cat ./+COMPACT_MANIFEST)
#   - or -
#       read_pkgng_origin `cat ./+COMPACT_MANIFEST`
#   - or -
#       read_pkgng_origin $data
#
#   Using double-quote encapsulation, you can optionally query multiple
#   packing-lists with the following example syntax:
#
#   Multiple arguments:
#       read_pkgng_origin \
#           "$(cat pkgA/+COMPACT_MANIFEST)" "$(cat pkgB/+COMPACT_MANIFEST)"
#   - or -
#       dataA=$(cat pkgA/+COMPACT_MANIFEST)
#       dataB=$(cat pkgB/+COMPACT_MANIFEST)
#       read_pkgng_origin "$dataA" "$dataB"
#
#   Finally, you can pipe the packing-list data directly into the function. See
#   below:
#
#   Handling PIPE data:
#       cat ./+COMPACT_MANIFEST | read_pkgng_origin
#   - or -
#       read_pkgng_origin < ./+COMPACT_MANIFEST
#
#   Only the first such "origin" field from any/all supplied packing-list data
#   is printed, with the exception of when multiple arguments are provided
#   (when the first "origin" field is printed for each argument, as-if each
#   argument was a full-and-complete packing-list from an independent package).
#
read_pkgng_origin()
{
    local origin

    if [ $# -gt 0 ]; then
        # Parse each argument as a full-and-complete packing-list
        while [ $# -gt 0 ]; do
            origin=
            manifest_read -r origin -i "$1"
            echo "$origin"
            shift 1
        done
        return $SUCCESS
    fi

    #
    # Expect PIPE data to be provided from STDIN (otherwise, as
    # expected, the user will be prompted to provide the input on
    # the command-line when `awk' is invoked
    #
    origin=
    manifest_read -r origin
    echo "$origin"
}

############################################################ MAIN

#
# Perform sanity checks
#
[ $# -gt 0 ] || usage

#
# Process command-line options
#
while getopts hoaf flag; do
    case "$flag" in
    h|\?) usage;;
    o) USE_ORIGIN_SUBDIR=1;;
    a) MOVE_PACKAGE=1;;
    f) FORCE=1;;
    esac
done
shift $(( $OPTIND - 1 ))

#
# Perform [another] sanity check
#
[ $# -gt 0 ] || usage

#
# Loop over each remaining package-archive argument(s)
#
while [ $# -gt 0 ]; do

    package="$1"
    shift 1

    printf "===> Unpacking \`%s'\n" "$package"

    case "$package" in
    *gz) ar_opt=z;;
    *bz) ar_opt=j;;
    *xz) ar_opt=J;;
    esac

    CAT=
    case "$ar_opt" in
    J) tar --help 2>&1 | awk '/-J/&&found++;END{exit !found}' ||
        CAT=xzcat ar_opt= ;;
    esac

    #
    # Extract packing-list from package
    #
    printf "Extracting +CONTENTS/+COMPACT_MANIFEST from package\n"
    CONTENTS_FORMAT=pkg_tools
    if [ "$CAT" ]; then
        CONTENTS="$( $CAT "$package" |
            tar ${ar_opt}xfO - +CONTENTS 2> /dev/null )"
    else
        CONTENTS="$( tar ${ar_opt}xfO "$package" +CONTENTS \
            2> /dev/null )"
    fi
    if [ ! "$CONTENTS" ]; then
        CONTENTS_FORMAT=pkgng
        if [ "$CAT" ]; then
            CONTENTS="$( $CAT "$package" |
                tar ${ar_opt}xfO - +COMPACT_MANIFEST \
                2> /dev/null )"
        else
            CONTENTS="$( tar ${ar_opt}xfO "$package" \
                +COMPACT_MANIFEST 2> /dev/null )"
        fi
    fi
    if [ ! "$CONTENTS" ]; then
        eprintf "ERROR: Unable to extract packing-list from package\n"
        eprintf "ERROR: Skipping package\n"
        continue
    fi

    #
    # Get the package_name
    #
    # NOTE: By truncating the ORIGIN field of the packing-list we can
    #       reliably determine the package's base name without having
    #       to resort to attempting to infer its name based on the
    #       package's file (which contains the version string). In
    #       addition, it gives us a more definitive way of storing our
    #       package sources (by ORIGIN, which indicates author and/or
    #       section, rather than by base-name only).
    # NOTE: For pkgng, the ORIGIN is the "origin" field in either +MANIFEST
    #       or +COMPACT_MANIFEST.
    #
    case "$CONTENTS_FORMAT" in
    pkg_tools)
        printf "Reading ORIGIN from packing-list: "
        ORIGIN=$( read_pkg_origin "$CONTENTS" ) ;;
    pkgng)
        printf "Reading origin from packing-list: "
        ORIGIN=$( read_pkgng_origin "$CONTENTS" ) ;;
    *)
        eprintf "ERROR: Unknown packing-list format!\n"
        eprintf "ERROR: Unable to unpack \`%s' (skipping)\n" "$package"
        continue
    esac
    if [ ! "$ORIGIN" ]; then
        printf "\n"
        eprintf "ERROR: No ORIGIN recorded for package\n"
        eprintf "ERROR: Unable to unpack \`%s'\n" "$package"

        read -p "Please provide a path to unpack to: " ORIGIN
        if [ ! "$ORIGIN" ]; then
            eprintf "ERROR: No path provided (skipping package)\n"
            continue
        fi
        USE_ORIGIN_SUBDIR=1
    else
        printf "%s\n" "$ORIGIN"
    fi

    #
    # Determine where we will unpack the package to.
    #
    if [ "$USE_ORIGIN_SUBDIR" ]; then
        DEST="$ORIGIN"
    else
        DEST="${ORIGIN##*/}"
    fi

    #
    # Make sure that the directory in which we are going to unpack to
    # doesn't already exist. If it does, issue an error and skip this
    # package (unless `-f' is passed).
    #
    # Otherwise, create the destination and proceed with unpacking.
    #
    printf "Creating package repository directory: "
    if [ -e "$DEST" ]; then
        printf "\n"
        eprintf "ERROR: Directory \`%s' already exists\n" "$DEST"
        if [ ! "$FORCE" ]; then
            eprintf "ERROR: Skipping package %s\n" \
                    "(use \`-f' to override)"
            continue
        else
            eprintf "ERROR: Proceeding anyway (\`-f' was passed)\n"
        fi
    fi
    if ! mkdir -p "$DEST"; then
        printf "\n"
        eprintf "ERROR: Could not create directory \`%s'\n" "$DEST"
        die "ERROR: Exiting"
    fi
    printf "%s\n" "$DEST"

    #
    # Unpack the package
    #
    printf "Unpacking...\n"
    if ! mkdir -p "$DEST/src"; then
        eprintf "ERROR: Could not create directory \`%s/src'\n" "$DEST"
        die "ERROR: Exiting"
    fi
    if [ "$CAT" ]; then
        $CAT "$package" | tar ${ar_opt}xfp - -C "$DEST/src"
    else
        tar ${ar_opt}xfp "$package" -C "$DEST/src"
    fi
    if [ $? -ne $SUCCESS ]; then
        eprintf "ERROR: Could not unpack \`%s' to \`%s/src'" \
            "$package" "$DEST"
        die "ERROR: Exiting"
    fi

    #
    # Copy the packing-list
    #
    case "$CONTENTS_FORMAT" in
    pkg_tools)
        printf "Generating \`PLIST' from \`+CONTENTS'...\n"
        if ! grep -v '^@comment[[:space:]]\{1,\}MD5:' \
           < "$DEST/src/+CONTENTS" > "$DEST/PLIST"; then
            eprintf "ERROR: Could not create \`PLIST'"
            die "ERROR: Exiting"
        fi
        ;;
    pkgng)
        printf "Generating \`MANIFEST' from \`+MANIFEST'...\n"
        # XXX unfinished XXX
        printf "WARNING! NOT ACTUALLY DONE YET\n"
        ;;
    esac

    #
    # Remove compiled packing-list
    #
    case "$CONTENTS_FORMAT" in
    pkg_tools)
        printf "Deleting \`+CONTENTS'...\n"
        rm -f "$DEST/src/+CONTENTS"
        ;;
    pkgng)
        printf "Deleting \`+MANIFEST' and \`+COMPACT_MANIFEST'...\n"
        printf "WARNING! NOT ACTUALLY DONE YET\n"
        #rm -f "$DEST/src/+MANIFEST" "$DEST/src/+COMPACT_MANIFEST"
        ;;
    esac

    #
    # Archive package
    #
    if [ "$MOVE_PACKAGE" = "1" ]; then
        echo "Archiving package..."
        if ! mkdir "$DEST/archive"; then
            eprintf "ERROR: Unable to create %s \`%s/archive'" \
                    "directory" "$DEST"
            die "ERROR: Exiting"
        fi
        if ! mv "$package" "$ORIGIN/archive"; then
            eprintf "ERROR: Unable to %s \`%s' to \`%s/archive'" \
                    "move" "$package" "$DEST"
            die "ERROR: Exiting"
        fi
    fi

    #
    # Extract skeleton directory into package repository
    #
    printf "Copying \`skel' structure into package repository...\n"
    tar co --exclude CVS -f - -C "$progdir/skel" . | tar xkvf - -C "$DEST"

    #
    # Move in the appropriate Makefile
    #
    printf "Adjusting for archive format...\n"
    case "$CONTENTS_FORMAT" in
    pkg_tools)
        case "$ar_opt" in
        z) rm -vf "$DEST/Makefile.ng"
           if [ ! -e "$DEST/Makefile" ] ||
              cmp "$DEST/Makefile" "$progdir/skel/Makefile"
           then
            mv -vf "$DEST/Makefile.old" "$DEST/Makefile"
           else
            rm -vf "$DEST/Makefile.old"
           fi
           ;;
        j) rm -vf "$DEST/Makefile.old" "$DEST/Makefile.ng" ;;
        esac
        ;;
    pkgng)
        rm -vf "$DEST/PLIST"
        rm -vf "$DEST/Makefile.old"
        if [ ! -e "$DEST/Makefile" ] ||
           cmp "$DEST/Makefile" "$progdir/skel/Makefile"
        then
            mv -vf "$DEST/Makefile.ng" "$DEST/Makefile"
        else
            rm -vf "$DEST/Makefile.ng"
        fi
        cmp "$DEST/src/+COMMENT" "$progdir/skel/src/+COMMENT" \
            2> /dev/null && rm -fv "$DEST/src/+COMMENT"
        cmp "$DEST/src/+DESC" "$progdir/skel/src/+DESC" 2> /dev/null &&
            rm -fv "$DEST/src/+DESC"
        ;;
    esac

    #
    # That's it (onto the next, back at the top).
    #
    printf "Done.\n"
done

################################################################################
# END
################################################################################
#
# $Header: //depot/bugatti/pkgbase/freebsd/unpack.sh#5 $
#
# $Copyright: 1999-2014 Devin Teske. All rights reserved. $
#
################################################################################