vaultscript

#!/bin/bash

if [ $# -eq 0 ]; then
    echo "No arguments supplied. Example: ./script.sh https://vimm.net/vault/GB"
    exit 1
fi

arg="$1"
console=$(echo "$arg" | awk -F '/' '{print $(NF)}')
echo "$console"
for letter in \# A B C D E F G H I J K L M N O P Q R S T U V W X Y Z; do
    # Construct letterpage
    if [ "$letter" = "#" ]; then
        letterpage="https://vimm.net/vault/?p=list&system=$console&section=number"
    else
        letterpage="https://vimm.net/vault/$console/$letter"
    fi

    echo
    echo "$letter: $letterpage"
    echo "======================="

    # Base URL (should be without trailing slash)
    baseURL="https://vimm.net"

    # find all gamepages in a letterpage
    for gamepage in $(curl -s "$letterpage" | grep 'a href="\/vault' | awk -F 'href="' '{print $2}' | cut -d '"' -f1 | grep -v '?p=' | grep -v "NES" | grep -v "$console" | grep -o '[[:digit:]]*'); do
        # Construct the gamepage URL
        gamepage="$baseURL/vault/$gamepage"

        skipit=false
        retries=1

        # Find the mediaID for this specific game page
        while true; do
            echo "Looking for mediaID into: $gamepage"
            id=$(curl -s "$gamepage" | grep 'var media = {"ID":' | awk -F ':' '{ print $2}' | awk -F ',' '{print $1}')
            if [ -z "$id" ]; then
                echo "\$id is empty. Retrying ...$retries"
                ((retries = retries + 1))

                # But ... if you re-tried 10 times already, then give up on this one
                if [[ "$retries" -gt 10 ]]; then
                    skipit=true
                    break
                fi
            else
                echo "FOUND! ID is: $id"
                break
            fi
        done

        # If you bailed out because the mediaID was not found, then proceed to the next gamepage
        if [ "$skipit" = true ]; then
            echo 'Giving up on this one...'
            echo "$id@$console@$gamepage" >>"$console.err"
            continue
        fi

        # Find Download URL
        skipit=false
        retries=1
        while true; do
            echo "Looking for download URL"
            URL=$(curl -s "$gamepage" | grep '<form action="//' | awk -F '<form action="//' '{ print $2 }' | awk -F '"' '{ print $1 }')
            if [ -z "$URL" ]; then
                echo "\$URL is empty. Retrying ...$retries"
                ((retries = retries + 1))

                # But ... if you re-tried 10 times already, then give up on this one
                if [[ "$retries" -gt 10 ]]; then
                    skipit=true
                    break
                fi
            else
                echo "FOUND! URL is: $URL"
                break
            fi
        done

        # If you bailed out because the download URL was not found, then proceed to the next gamepage
        if [ "$skipit" = true ]; then
            echo 'Giving up on this one...'
            echo
            echo
            echo "$id@$console@$gamepage" >>"$console.err"
            continue
        fi

        # Constrcut the download URL
        # e.g. URL="https://download.vimm.net/download/?mediaId=$id"
        URL="https://$URL?mediaId=$id"

        echo "Looking for filename into: $URL"

        # Find the filename  (Check there is 404, so skip it)
        skipit=false
        retries=1
        while true; do
            filename=$(curl -s --max-time 5 -D - \
                -H "Referer: https://vimm.net/" \
                -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36" \
                "$URL" | grep "Content-Disposition" | awk -F 'filename="' '{ print $2 }' | awk -F '"' '{ print $1 }' 2>/dev/null)
            if [ -z "$filename" ]; then
                echo "\$filename is empty. Retrying ... $retries"
                ((retries = retries + 1))

                # But ... if you re-tried 10 times already, then give up on this one
                if [[ "$retries" -gt 10 ]]; then
                    skipit=true
                    break
                fi
            else
                echo "FOUND! Filename is: $filename"
                break
            fi
        done

        #  If you bailed out because the filename was not found, then proceed to the next gamepage
        if [ "$skipit" = true ]; then
            echo 'Giving up on this one...'
            echo
            echo
            echo "$id@$console@$gamepage@$URL@$filename" >>"$console.txt"
            continue
        fi

        # Download the rom
        skipit=false
        retries=1
        while true; do
            if curl \
                -H "Referer: https://vimm.net/" \
                -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36" \
                "$URL" --output "$filename" --progress-bar; then
                echo "Start downloading $filename..."
            fi

            # Check if the file integrity is correct
            ZIP=false
            SEVEN=false

            if test -f "$filename"; then
                echo "Check integrity..."
                if file "$filename" | grep 'Zip archive data\|7-zip archive data' &>/dev/null; then
                    echo "VALID!"
                    if file "$filename" | grep 'Zip archive data' &>/dev/null; then
                        echo "Compression Type: Zip archive"
                        ZIP=true
                    fi
                    if file "$filename" | grep '7-zip archive data' &>/dev/null; then
                        echo "Compression Type: 7z archive"
                        SEVEN=true
                    fi
                    if [ "$ZIP" = true ] || [ "$SEVEN" = true ]; then
                        echo "It's a known compressed format"
                    else
                        echo "I don't know this compression format"
                        ((retries = 11))
                        break
                    fi

                    # All fine, break the loop to proceed to the next step that is extraction
                    break
                else
                    # PROBLEM: In case compressed archive is damaged, retry 10 times or give up
                    echo "INVALID! $filename is not a valid Zip/7z archive. Retrying ..."
                    ((retries = retries + 1))
                    # But ... if you re-tried 10 times already, then give up on this one
                    if [[ "$retries" -gt 10 ]]; then
                        skipit=true
                        break
                    else
                        continue # retry (this will re-download the file)
                    fi
                fi
            else
                # PROBLEM: In case download failed, retry 10 times or give up
                echo "$filename has not been downloaded. Retrying ... $retries"
                ((retries = retries + 1))

                # But ... if you re-tried 10 times already, then give up on this one
                if [[ "$retries" -gt 10 ]]; then
                    skipit=true
                    break
                fi

                continue # Retry (this will re-download the file)

            fi
        done

        # If you bailed out because the zip/7z compressed archive is invalid or failed to download, then proceed to the next GAMELINK
        if [ "$skipit" = true ]; then
            echo 'Giving up on this one...'
            echo "Deleting filename: $filename"
            rm "$filename" 2>/dev/null
            echo "Writing logs at $console.err"
            echo "$id@$console@$gamepage@$URL@$filename" >>"$console.err"
            echo "Proceeding to the next game ..."
            echo
            echo
            echo "================================================="
            continue # Proceed to the next Game from the outer loop
        fi

        # ------------ #

        # Extract the rom
        skipit=false
        retries=1
        while true; do
            folder=$(date)
            echo "Trying to extract..."
            echo "Creating a ./tmp directory and moving $filename into it"
            mkdir ./tmp
            mv "$filename" ./tmp
        echo "tmp created and file moved into it"

            if [ "$SEVEN" = true ]; then
                if 7z x "./tmp/$filename" -o./tmp; then
                    echo "Extracted!"
                    folder=$(echo "$filename" | awk -F '.7z' '{print $1}')
                    echo "Deleting the compressed 7z archive: $filename"
                    rm "./tmp/$filename"
                    echo "Deleted!"
                    echo "Renaming the extracted folder ./tmp to $folder"
                    mv ./tmp "$folder"
                    echo "Renamed!"
                else
                    echo "FAILED! Cannot un7zip the $filename"
                    echo "Deleting the ./tmp directory"
                    rm -rf tmp
                    echo "Deleted!"
                    echo "Retrying ... $retries"
                    ((retries = retries + 1))
                    # But ... if you re-tried 10 times already, then give up on this one
                    if [[ "$retries" -gt 10 ]]; then
                        skipit=true
                        break
                    else
                        continue # retry (this will re-extract the file)
                    fi
                fi
            fi

            if [ "$ZIP" = true ]; then
                if unzip -o "tmp/$filename" -d ./tmp; then
                    echo "Extracted!"
                    folder=$(echo "$filename" | awk -F '.zip' '{print $1}')
                    echo "Deleting the compressed zip archive: $filename"
                    rm "tmp/$filename"
                    echo "Deleted!"
                    echo "Renaming the extracted folder ./tmp to $folder"
                    mv ./tmp "$folder"
                    echo "Renamed!"
                else
                    echo "FAILED! Cannot unzip the $filename"
                    echo "Deleting the ./tmp directory"
                    rm -rf tmp
                    echo "Deleted!"
                    echo "Retrying ... $retries"
                    ((retries = retries + 1))
                    # But ... if you re-tried 10 times already, then give up on this one
                    if [[ "$retries" -gt 10 ]]; then
                        skipit=true
                        break
                    else
                        continue # retry (this will re-extract the file)
                    fi
                fi
            fi

            # All fine, break the loop to proceed to the next game
            break
        done

        # If you bailed out because the zip/7z compressed archive is invalid or failed to download, then proceed to the next GAMELINK
        if [ "$skipit" = true ]; then
            echo 'Giving up on this one...'
            echo "Deleting filename: $filename"
            rm "$filename" 2>/dev/null
            echo "Writing logs at $console.err"
            echo "$id@$console@$gamepage@$URL@$filename" >>"$console.err"
            echo "Proceeding to the next game ..."
            echo
            echo
            echo "================================================="
            continue # Proceed to the next Game from the outer loop
        fi

        # ------------ #
        echo "DONE! Game $filename with $id has been downloaded and extracted into $folder!"
        echo
        echo
    done
    echo "---"
done