Subversion Repositories DevTools

#!/bin/bash
#   This script will restore a package from S3 into dpkg_archive
#   See savePkgToS3.sh
#
#   The source bucket is: aupera-dpkg-quarantine
#   
#   The package version is expeced to be tar-zipped
#
#   The resulatant tar-zip will be transferred to S3
#
#       Reduced redundancy is used
#
#       Credentials: s3_dpkg user 
#
#       Usage:  getPkgFromS3.sh /PathTo/pkgName/pkgVersion
#
function doHelp {
cat <<endOfHelp
    Command: getPkgFromS3

    This program will retrieve tar-zip Package/Version from an Amazon S3 bucket
    and store it into a subdirectory specified

    Options
    -h, --help              - Display this message
    -v, --verbose           - Increase verbosity
    -q, --quiet             - No status messages
    -b, --bucket=name       - Specifies the name of the target bucket
    -p, --path=PVPath       - Specifies the path to the Package-Version to restore
    -f, --force             - Delete target and force package download
    -k, --key=keyVar        - Name of the EnvVar that conains the AWS key
                              Default is AWSKEY
    -s, --secret=secretVar  - Name of the EnvVar that conains the AWS secret
                              Default is AWSSECRET
    -t, --test              - Do not copy, do not request Glacier restore

   Example:
    getPkgFromS3.sh -p \$GBE_DPKG/AcceptanceTestFramework/1.0.10000.cr
endOfHelp
}

#
#   Init defaults
#
ProgName=getPkgFromS3
awsKeyVar=AWSKEY
awsSecretVar=AWSSECRET
bucket=aupera-dpkg-quarantine
verbose=1
forceDelete=0
doOps=true

# Note that we use `"$@"' to let each command-line parameter expand to a 
# separate word. The quotes around `$@' are essential!
# We need TEMP as the `eval set --' would nuke the return value of getopt.
TEMP=$( getopt -n ${ProgName} -o qfvhb:p:k:s:t --long quiet,force,verbose,help,bucket:,path:,key:,secret:,test -- "$@" )

if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi

# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"

while true ; do
    case "$1" in
        -h|--help)    doHelp; exit 0;;
        -f|--force)   let forceDelete++; shift 1;;
        -q|--quiet)   verbose=0; shift 1;;
        -v|--verbose) let verbose++; shift 1;;
        -b|--bucket)  bucket="$2" ; shift 2 ;;
        -p|--path)    dpkgPath="$2" ; shift 2 ;;
        -k|--key)     awsKeyVar="$2" ; shift 2 ;;
        -s|--secret)  awsSecretVar="$2" ; shift 2 ;;
        -t|--test)   doOps=false ; shift 1 ;;
        --) shift ; break ;;
        *) echo "Internal error!" ; exit 1 ;;
    esac
done

if [ $verbose -gt 1 ] ; then
    echo bucket      :$bucket
    echo dpkgPath    :$dpkgPath
    echo awsKeyVar   :$awsKeyVar
    echo awsSecretVar:$awsSecretVar

    echo "Remaining arguments:"
    for arg do echo '--> '"\`$arg'" ; done
fi

: ${bucket:?No Bucket Specified}
: ${dpkgPath:?No Package Version Path}
: ${awsKeyVar:?No AWS Key specified}
: ${awsSecretVar:?No AWS Secret specified}

#
#   The KEY and the Secret are passed via EnvVars
#   The name of the vars are passed on the command line
#
aws_access_key_id=${!awsKeyVar}
aws_secret_access_key=${!awsSecretVar}

: ${aws_access_key_id:?No AWS Key found}
: ${aws_secret_access_key:?No AWS Secret found}

if [ $verbose -gt 1 ] ; then
    echo aws_access_key_id:$aws_access_key_id
    echo aws_secret_access_key:$aws_secret_access_key
fi

#
#   Determine
#       dpkgPath    - Cleanup the user arg
#       pkgBase     - Base of the package
#       pkgName     - Package Name
#       pkgVer      - Package Version
#

dpkgPath=${dpkgPath%/}

pkgBase=${dpkgPath%/*}
pkgBase=${pkgBase%/*}

pkgVer=${dpkgPath##*/}

pkgName=${dpkgPath%/*}
pkgName=${pkgName##*/}

if [ $verbose -gt 0 ] ; then
    echo Bucket  : $bucket
    echo pkgBase : $pkgBase
    echo pkgName : $pkgName
    echo pkgVer  : $pkgVer
fi

if [  ! -d $pkgBase ] ; then
    echo "Error: Package archive base does not exist: $pkgBase"
    exit 1
fi

# Force version deletion - more for test.
if [ $forceDelete -gt 0 ] ; then 
    [ $verbose -gt 0 ] && echo "Force removal of $pkgName/$pkgVer"
    if [ -d $dpkgPath ] ; then
        if $doOps ; then
            chmod -R +w $dpkgPath
            rm -rf $dpkgPath
        else
            echo "TestMode: Did not remove: $pkgName/$pkgVer"
        fi     
    fi
fi

if [  -d $dpkgPath ] ; then
    echo "Error: Target PV Path already exists: $dpkgPath"
    echo "TestMode: $doOps"
    if ! $doOps ; then
        echo "TestMode: Error ignored"
    else
        exit 1
    fi
fi

#
#   Create the source file name
#   Format: Quarantined/PkgName_PkgVersion.tgz
file="Quarantined/${pkgName}_${pkgVer}.tgz"

# Basic transfer requirements
resource="/${bucket}/${file}"
dateValue=$(date -R)


#############################################################
#   Fetch file info, just to be sure that the file is there
#   Get data about the file
#
# Calculate the HEAD signature.
#   Note the need for a triple \n
#   Is that because there is no contentType ?
#
stringToSign="HEAD\n\n\n${dateValue}\n${resource}"
signature=$(
    echo -en "${stringToSign}" |
    openssl sha1 -hmac "${aws_secret_access_key}" -binary |
    base64
)

#set -x
fileTest=0
[ $verbose -gt 1 ] && echo "Testing file presence: ${file}"
results=$(curl -I -X HEAD \
        -s \
         --insecure \
        -H "Host: ${bucket}.s3.amazonaws.com" \
        -H "Date: ${dateValue}" \
        -H "Authorization: AWS ${aws_access_key_id}:${signature}" \
        "https://${bucket}.s3.amazonaws.com/${file}" \
        )
if [ $verbose -gt 2 ] ; then
    echo "Testing file response"
    echo "$results" | fold -w100 | awk '{print "    " $0}'
fi

if [[ "$results" =~ "HTTP/1.1 200 OK" ]]; then
    fileTest=1
fi

# Display results
if [ $fileTest -gt 0 ]; then
    if [ $verbose -gt 1 ] ; then
        echo "${ProgName}: Package Version Exists: $pkgName/$pkgVer" 
    fi
else
    echo "${ProgName}: Error cannot access $pkgName/$pkgVer in S3 bucket ${bucket}"
    exit 1
fi

# Ensure correct storage class
#   STANDARD_IA - Good
#   REDUCED_REDUNDANCY - Good
#   GLACIER - May be good
#       If it has been restored, then its good
#       Else its an error
#       Restored status is held in x-amz-restore attribute
#           When it exists the file is restored or is being restored
#           When the 'ongoing-request' is false, then it has been restored
canGet=false
mustRestore=false
message=""
storageClass=$( echo $results | tr '\r' '\n' | awk -F: '/x-amz-storage-class/{print $2}')
[ $verbose -gt 0 ] && echo storageClass : $storageClass
if [[ "$storageClass" != " REDUCED_REDUNDANCY" &&  "$storageClass" != " STANDARD_IA" ]]; then
    if [[ "$storageClass" == " GLACIER" ]]; then
        [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer has been transferred to glacier"
        glacierState="In Glacier"
        RestoreData=$( echo $results | tr '\r' '\n' | awk -F: '/x-amz-restore/{print $2,$3,$4,$5,$6,$7,$8,$9}')
        if [ -n "$RestoreData" ] ; then
            [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer Restored progress: $RestoreData"
            ongoingRequest=$( echo "$RestoreData" | sed -n 's/.*ongoing-request="\([^"]*\)".*'/\\1/p)
            expiryDate=$( echo "$RestoreData" | sed -n 's/.*expiry-date="\([^"]*\)".*'/\\1/p )
            [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer Restored State: $ongoingRequest"
            [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer Restored Expiry Date: $expiryDate"
            if [ "$ongoingRequest" != "false" ] ; then
                glacierState="Restore in progress"
                message="Restore not complete: ongoing-request=$ongoingRequest"
            else
                glacierState="Restored"
                canGet=true
            fi
        else
            message="Incorrect storage class: '$storageClass'"
            mustRestore="true"        
        fi
        [ $verbose -gt 0 ] && echo "glacierState : $glacierState"
    else
        message="Unhandled storage class: '$storageClass'"        
    fi
else
    canGet=true
fi

##############################################################
#   Initiate a Restore from Glacier
#
if $mustRestore ; then
    if $doOps ; then
        [ $verbose -gt 0 ] && echo "Initiate restore from Glacier $pkgName/$pkgVer"

        # Generate a Restore Request
        #   And an MD5 over the data
        tmpData=$( mktemp )
        echo > $tmpData '<RestoreRequest>'
        echo >>$tmpData '  <Days>1</Days>'
        echo >>$tmpData '  <GlacierJobParameters>'
        echo >>$tmpData '    <Tier>Standard</Tier>'
        echo >>$tmpData '  </GlacierJobParameters>'
        echo >>$tmpData '</RestoreRequest>'

        md5Value=$(
            openssl md5 -binary $tmpData |
            base64
        )

        # Generate the request signature
        contentType="application/xml"
        stringToSign="POST\n${md5Value}\n${contentType}\n${dateValue}\n${resource}?restore"
        signature=$(
            echo -en "${stringToSign}" |
            openssl sha1 -hmac "${aws_secret_access_key}" -binary |
            base64
        )

        #set -x
        tmpResult=$( mktemp )
        results=$(curl -s -w '%{http_code}' -o $tmpResult\
             --insecure \
             --data @$tmpData \
            -H "Host: ${bucket}.s3.amazonaws.com" \
            -H "Date: ${dateValue}" \
            -H "Content-MD5: ${md5Value}" \
            -H "Authorization: AWS ${aws_access_key_id}:${signature}" \
            -H "Content-Type: ${contentType}" \
            "https://${bucket}.s3.amazonaws.com/${file}?restore"
            )

        canGet=false
        if [[ "$results" == "200" ]] ; then
            canGet=true
        elif [[ "$results" == "202" ]] ; then
            message="Restore from Glacier initiated"
        elif [[ "$results" == "409" ]] ; then
            message="Restore already in progress"
        elif [[ "$results" == "503" ]] ; then
            message="Glacier expedited retrievals are currently not available"
        else
            mesage="Error previously reported"
            echo "Glacier restore error code: $results"
            tail -n+2 $tmpResult | fold -w100 | awk '{print "    " $0}'
        fi                              
        set +x
        rm -rf $tmpData $tmpResult

    else
        echo "TestMode: Do not request Glacier Restore"
    fi
fi

if ! $canGet; then
        echo "${ProgName}: Error cannot access $pkgName/$pkgVer. $message"
        echo "Resource: $resource"
        exit 1
fi

#############################################################
# GET!

# Calculate the signature.
#   Note the need for a triple \n
#   Needed becase there is no content type
stringToSign="GET\n\n\n${dateValue}\n${resource}"
signature=$(
    echo -en "${stringToSign}" |
    openssl sha1 -hmac "${aws_secret_access_key}" -binary |
    base64
)

#echo dateValue: ${dateValue} 
#echo stringToSign: ${stringToSign}
#echo signature: ${signature}
#exit 1

mkdir -p $dpkgPath
if [  ! -d $dpkgPath ] ; then
    echo "Error: Could not create: $dpkgPath"
    exit 1
fi

if $doOps ; then
    [ $verbose -gt 0 ] && echo "Transfer $pkgName/$pkgVer from bucket $bucket"
    #set -x
        curl -s \
            -X GET \
             --insecure \
            -H "Host: ${bucket}.s3.amazonaws.com" \
            -H "Date: ${dateValue}" \
            -H "Authorization: AWS ${aws_access_key_id}:${signature}" \
            "https://${bucket}.s3.amazonaws.com/${file}" \
            | tar -xz --strip=2 -C $dpkgPath

    if [ ! -f "$dpkgPath/descpkg" ] ; then
        echo "Error: Target PV did not populate as expected: $dpkgPath"

        # Remove if not correctly formatted
        chmod -R +w $dpkgPath
        rm -rf $dpkgPath
        exit 1
    else
        # Package extracted
        # Touch the top directory so that it will not be quarantined immediatly
        chmod +w "$dpkgPath"
        chmod -w "$dpkgPath"
    fi
else
    echo "TestNode: Did not transfer $pkgName/$pkgVer from bucket $bucket"
fi

[ $verbose -gt 0 ] && echo "Transfer complete"