Subversion Repositories DevTools

#!/bin/bash

#   This script will restore a package from S3 into dpkg_archive

#   See savePkgToS3.sh

#

#   The source bucket is: aupera-dpkg-quarantine

#   

#   The package version is expeced to be tar-zipped

#

#   The resulatant tar-zip will be transferred to S3

#

#       Reduced redundancy is used

#

#       Credentials: s3_dpkg user 

#

#       Usage:  getPkgFromS3.sh /PathTo/pkgName/pkgVersion

#

function doHelp {

cat <<endOfHelp

    Command: getPkgFromS3

    This program will retrieve tar-zip Package/Version from an Amazon S3 bucket

    and store it into a subdirectory specified

    Options

    -h, --help              - Display this message

    -v, --verbose           - Increase verbosity

    -q, --quiet             - No status messages

    -b, --bucket=name       - Specifies the name of the target bucket

    -p, --path=PVPath       - Specifies the path to the Package-Version to restore

    -f, --force             - Delete target and force package download

    -k, --key=keyVar        - Name of the EnvVar that conains the AWS key

                              Default is AWSKEY

    -s, --secret=secretVar  - Name of the EnvVar that conains the AWS secret

                              Default is AWSSECRET

    -t, --test              - Do not copy, do not request Glacier restore

   Example:

    getPkgFromS3.sh -p \$GBE_DPKG/AcceptanceTestFramework/1.0.10000.cr

endOfHelp

}

#

#   Init defaults

#

ProgName=getPkgFromS3

awsKeyVar=AWSKEY

awsSecretVar=AWSSECRET

bucket=aupera-dpkg-quarantine

verbose=1

forceDelete=0

doOps=true

# Note that we use `"$@"' to let each command-line parameter expand to a 

# separate word. The quotes around `$@' are essential!

# We need TEMP as the `eval set --' would nuke the return value of getopt.

TEMP=$( getopt -n ${ProgName} -o qfvhb:p:k:s:t --long quiet,force,verbose,help,bucket:,path:,key:,secret:,test -- "$@" )

if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi

# Note the quotes around `$TEMP': they are essential!

eval set -- "$TEMP"

while true ; do

    case "$1" in

        -h|--help)    doHelp; exit 0;;

        -f|--force)   let forceDelete++; shift 1;;

        -q|--quiet)   verbose=0; shift 1;;

        -v|--verbose) let verbose++; shift 1;;

        -b|--bucket)  bucket="$2" ; shift 2 ;;

        -p|--path)    dpkgPath="$2" ; shift 2 ;;

        -k|--key)     awsKeyVar="$2" ; shift 2 ;;

        -s|--secret)  awsSecretVar="$2" ; shift 2 ;;

        -t|--test)   doOps=false ; shift 1 ;;

        --) shift ; break ;;

        *) echo "Internal error!" ; exit 1 ;;

    esac

done

if [ $verbose -gt 1 ] ; then

    echo bucket      :$bucket

    echo dpkgPath    :$dpkgPath

    echo awsKeyVar   :$awsKeyVar

    echo awsSecretVar:$awsSecretVar

    echo "Remaining arguments:"

    for arg do echo '--> '"\`$arg'" ; done

fi

: ${bucket:?No Bucket Specified}

: ${dpkgPath:?No Package Version Path}

: ${awsKeyVar:?No AWS Key specified}

: ${awsSecretVar:?No AWS Secret specified}

#

#   The KEY and the Secret are passed via EnvVars

#   The name of the vars are passed on the command line

#

aws_access_key_id=${!awsKeyVar}

aws_secret_access_key=${!awsSecretVar}

: ${aws_access_key_id:?No AWS Key found}

: ${aws_secret_access_key:?No AWS Secret found}

if [ $verbose -gt 1 ] ; then

    echo aws_access_key_id:$aws_access_key_id

    echo aws_secret_access_key:$aws_secret_access_key

fi

#

#   Determine

#       dpkgPath    - Cleanup the user arg

#       pkgBase     - Base of the package

#       pkgName     - Package Name

#       pkgVer      - Package Version

#

dpkgPath=${dpkgPath%/}

pkgBase=${dpkgPath%/*}

pkgBase=${pkgBase%/*}

pkgVer=${dpkgPath##*/}

pkgName=${dpkgPath%/*}

pkgName=${pkgName##*/}

if [ $verbose -gt 0 ] ; then

    echo Bucket  : $bucket

    echo pkgBase : $pkgBase

    echo pkgName : $pkgName

    echo pkgVer  : $pkgVer

fi

if [  ! -d $pkgBase ] ; then

    echo "Error: Package archive base does not exist: $pkgBase"

    exit 1

fi

# Force version deletion - more for test.

if [ $forceDelete -gt 0 ] ; then 

    [ $verbose -gt 0 ] && echo "Force removal of $pkgName/$pkgVer"

    if [ -d $dpkgPath ] ; then

        if $doOps ; then

            chmod -R +w $dpkgPath

            rm -rf $dpkgPath

        else

            echo "TestMode: Did not remove: $pkgName/$pkgVer"

        fi     

    fi

fi

if [  -d $dpkgPath ] ; then

    echo "Error: Target PV Path already exists: $dpkgPath"

    echo "TestMode: $doOps"

    if ! $doOps ; then

        echo "TestMode: Error ignored"

    else

        exit 1

    fi

fi

#

#   Create the source file name

#   Format: Quarantined/PkgName_PkgVersion.tgz

file="Quarantined/${pkgName}_${pkgVer}.tgz"

# Basic transfer requirements

resource="/${bucket}/${file}"

dateValue=$(date -R)

#############################################################

#   Fetch file info, just to be sure that the file is there

#   Get data about the file

#

# Calculate the HEAD signature.

#   Note the need for a triple \n

#   Is that because there is no contentType ?

#

stringToSign="HEAD\n\n\n${dateValue}\n${resource}"

signature=$(

    echo -en "${stringToSign}" |

    openssl sha1 -hmac "${aws_secret_access_key}" -binary |

    base64

)

#set -x

fileTest=0

[ $verbose -gt 1 ] && echo "Testing file presence: ${file}"

results=$(curl -I -X HEAD \

        -s \

         --insecure \

        -H "Host: ${bucket}.s3.amazonaws.com" \

        -H "Date: ${dateValue}" \

        -H "Authorization: AWS ${aws_access_key_id}:${signature}" \

        "https://${bucket}.s3.amazonaws.com/${file}" \

        )

if [ $verbose -gt 2 ] ; then

    echo "Testing file response"

    echo "$results" | fold -w100 | awk '{print "    " $0}'

fi

if [[ "$results" =~ "HTTP/1.1 200 OK" ]]; then

    fileTest=1

fi

# Display results

if [ $fileTest -gt 0 ]; then

    if [ $verbose -gt 1 ] ; then

        echo "${ProgName}: Package Version Exists: $pkgName/$pkgVer" 

    fi

else

    echo "${ProgName}: Error cannot access $pkgName/$pkgVer in S3 bucket ${bucket}"

    exit 1

fi

# Ensure correct storage class

#   STANDARD_IA - Good

#   REDUCED_REDUNDANCY - Good

#   GLACIER - May be good

#       If it has been restored, then its good

#       Else its an error

#       Restored status is held in x-amz-restore attribute

#           When it exists the file is restored or is being restored

#           When the 'ongoing-request' is false, then it has been restored

canGet=false

mustRestore=false

message=""

storageClass=$( echo $results | tr '\r' '\n' | awk -F: '/x-amz-storage-class/{print $2}')

[ $verbose -gt 0 ] && echo storageClass : $storageClass

if [[ "$storageClass" != " REDUCED_REDUNDANCY" &&  "$storageClass" != " STANDARD_IA" ]]; then

    if [[ "$storageClass" == " GLACIER" ]]; then

        [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer has been transferred to glacier"

        glacierState="In Glacier"

        RestoreData=$( echo $results | tr '\r' '\n' | awk -F: '/x-amz-restore/{print $2,$3,$4,$5,$6,$7,$8,$9}')

        if [ -n "$RestoreData" ] ; then

            [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer Restored progress: $RestoreData"

            ongoingRequest=$( echo "$RestoreData" | sed -n 's/.*ongoing-request="\([^"]*\)".*'/\\1/p)

            expiryDate=$( echo "$RestoreData" | sed -n 's/.*expiry-date="\([^"]*\)".*'/\\1/p )

            [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer Restored State: $ongoingRequest"

            [ $verbose -gt 1 ] && echo "$pkgName/$pkgVer Restored Expiry Date: $expiryDate"

            if [ "$ongoingRequest" != "false" ] ; then

                glacierState="Restore in progress"

                message="Restore not complete: ongoing-request=$ongoingRequest"

            else

                glacierState="Restored"

                canGet=true

            fi

        else

            message="Incorrect storage class: '$storageClass'"

            mustRestore="true"        

        fi

        [ $verbose -gt 0 ] && echo "glacierState : $glacierState"

    else

        message="Unhandled storage class: '$storageClass'"        

    fi

else

    canGet=true

fi

##############################################################

#   Initiate a Restore from Glacier

#

if $mustRestore ; then

    if $doOps ; then

        [ $verbose -gt 0 ] && echo "Initiate restore from Glacier $pkgName/$pkgVer"

        # Generate a Restore Request

        #   And an MD5 over the data

        tmpData=$( mktemp )

        echo > $tmpData '<RestoreRequest>'

        echo >>$tmpData '  <Days>1</Days>'

        echo >>$tmpData '  <GlacierJobParameters>'

        echo >>$tmpData '    <Tier>Standard</Tier>'

        echo >>$tmpData '  </GlacierJobParameters>'

        echo >>$tmpData '</RestoreRequest>'

        md5Value=$(

            openssl md5 -binary $tmpData |

            base64

        )

        # Generate the request signature

        contentType="application/xml"

        stringToSign="POST\n${md5Value}\n${contentType}\n${dateValue}\n${resource}?restore"

        signature=$(

            echo -en "${stringToSign}" |

            openssl sha1 -hmac "${aws_secret_access_key}" -binary |

            base64

        )

        #set -x

        tmpResult=$( mktemp )

        results=$(curl -s -w '%{http_code}' -o $tmpResult\

             --insecure \

             --data @$tmpData \

            -H "Host: ${bucket}.s3.amazonaws.com" \

            -H "Date: ${dateValue}" \

            -H "Content-MD5: ${md5Value}" \

            -H "Authorization: AWS ${aws_access_key_id}:${signature}" \

            -H "Content-Type: ${contentType}" \

            "https://${bucket}.s3.amazonaws.com/${file}?restore"

            )

        canGet=false

        if [[ "$results" == "200" ]] ; then

            canGet=true

        elif [[ "$results" == "202" ]] ; then

            message="Restore from Glacier initiated"

        elif [[ "$results" == "409" ]] ; then

            message="Restore already in progress"

        elif [[ "$results" == "503" ]] ; then

            message="Glacier expedited retrievals are currently not available"

        else

            mesage="Error previously reported"

            echo "Glacier restore error code: $results"

            tail -n+2 $tmpResult | fold -w100 | awk '{print "    " $0}'

        fi                              

        set +x

        rm -rf $tmpData $tmpResult

    else

        echo "TestMode: Do not request Glacier Restore"

    fi

fi

if ! $canGet; then

        echo "${ProgName}: Error cannot access $pkgName/$pkgVer. $message"

        echo "Resource: $resource"

        exit 1

fi

#############################################################

# GET!

# Calculate the signature.

#   Note the need for a triple \n

#   Needed becase there is no content type

stringToSign="GET\n\n\n${dateValue}\n${resource}"

signature=$(

    echo -en "${stringToSign}" |

    openssl sha1 -hmac "${aws_secret_access_key}" -binary |

    base64

)

#echo dateValue: ${dateValue} 

#echo stringToSign: ${stringToSign}

#echo signature: ${signature}

#exit 1

mkdir -p $dpkgPath

if [  ! -d $dpkgPath ] ; then

    echo "Error: Could not create: $dpkgPath"

    exit 1

fi

if $doOps ; then

    [ $verbose -gt 0 ] && echo "Transfer $pkgName/$pkgVer from bucket $bucket"

    #set -x

        curl -s \

            -X GET \

             --insecure \

            -H "Host: ${bucket}.s3.amazonaws.com" \

            -H "Date: ${dateValue}" \

            -H "Authorization: AWS ${aws_access_key_id}:${signature}" \

            "https://${bucket}.s3.amazonaws.com/${file}" \

            | tar -xz --strip=2 -C $dpkgPath

    if [ ! -f "$dpkgPath/descpkg" ] ; then

        echo "Error: Target PV did not populate as expected: $dpkgPath"

        # Remove if not correctly formatted

        chmod -R +w $dpkgPath

        rm -rf $dpkgPath

        exit 1

    else

        # Package extracted

        # Touch the top directory so that it will not be quarantined immediatly

        chmod +w "$dpkgPath"

        chmod -w "$dpkgPath"

    fi

else

    echo "TestNode: Did not transfer $pkgName/$pkgVer from bucket $bucket"

fi

[ $verbose -gt 0 ] && echo "Transfer complete"