Subversion Repositories DevTools

#!/bin/bash

#   This script will quarantine a package-vesrion into S3 storage

#   The target bucket is: auawsaddp001

#   

#   The package vesrion will be tar-zipped

#   without the dpkg_archive prefix

#

#   The resulatant tar-zip will be transferred to S3

#

#       Reduced redundancy is used

#

#       Credentials: s3_dpkg user 

#

#       Usage:  ./savePkg /PathTo/pkgName/pkgVersion

#

function doHelp {

cat <<endOfHelp

    Command: savePkgToS3

    This program will tar-zip a dpkg_archive Package/Version and store it

    into an Amazon S3 bucket in a subdirectory called 'Quarantine'

    Options

    -h, --help              - Display this message

    -v, --verbose           - Increase verbosity

    -b, --bucket=name       - Specifies the name of the target bucket

    -p, --path=PVPath       - Specifies the path to the Package-Version to save

    -k, --key=keyVar        - Name of the EnvVar that conains the AWS key

                              Default is AWSKEY

    -s, --secret=secretVar  - Name of the EnvVar that conains the AWS secret

                              Default is AWSSECRET

endOfHelp

}

#

#   Init defaults

#

ProgName=savePkgtoS3

awsKeyVar=AWSKEY

awsSecretVar=AWSSECRET

verbose=0

# Note that we use `"$@"' to let each command-line parameter expand to a 

# separate word. The quotes around `$@' are essential!

# We need TEMP as the `eval set --' would nuke the return value of getopt.

TEMP=$( getopt -n ${ProgName} -o vhb:p:k:s: --long verbose,help,bucket:,path:,key:,secret: -- "$@" )

if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi

# Note the quotes around `$TEMP': they are essential!

eval set -- "$TEMP"

while true ; do

	case "$1" in

		-h|--help)    doHelp; exit 0;;

		-v|--verbose) let verbose++; shift 1;;

		-b|--bucket)  bucket="$2" ; shift 2 ;;

		-p|--path)    dpkgPath="$2" ; shift 2 ;;

		-k|--key)     awsKeyVar="$2" ; shift 2 ;;

		-s|--secret)  awsSecretVar="$2" ; shift 2 ;;

		--) shift ; break ;;

		*) echo "Internal error!" ; exit 1 ;;

	esac

done

if [ $verbose -gt 1 ] ; then

    echo bucket      :$bucket

    echo dpkgPath    :$dpkgPath

    echo awsKeyVar   :$awsKeyVar

    echo awsSecretVar:$awsSecretVar

    echo "Remaining arguments:"

    for arg do echo '--> '"\`$arg'" ; done

fi

: ${bucket:?No Bucket Specified}

: ${dpkgPath:?No Package Version Path}

: ${awsKeyVar:?No AWS Key specified}

: ${awsSecretVar:?No AWS Secret specified}

#

#   The KEY and the Secret are passed via EnvVars

#   The name of the vars are passed on the command line

#

aws_access_key_id=${!awsKeyVar}

aws_secret_access_key=${!awsSecretVar}

: ${aws_access_key_id:?No AWS Key found}

: ${aws_secret_access_key:?No AWS Secret found}

if [ $verbose -gt 1 ] ; then

    echo aws_access_key_id:$aws_access_key_id

    echo aws_secret_access_key:$aws_secret_access_key

fi

#

#   Determine

#       dpkgPath    - Cleanup the user arg

#       pkgBase     - Base of the package

#       pkgName     - Package Name

#       pkgVer      - Package Version

#

dpkgPath=${dpkgPath%/}

pkgBase=${dpkgPath%/*}

pkgBase=${pkgBase%/*}

pkgVer=${dpkgPath##*/}

pkgName=${dpkgPath%/*}

pkgName=${pkgName##*/}

if [ $verbose -gt 1 ] ; then

    echo pkgBase:$pkgBase

    echo pkgName:pkgName

    echo pkgVer:$pkgVer

fi

if [ ! -d $dpkgPath ] ; then

    echo "Error: PVPath does not address a directory"

    exit 1

fi

#

#   Create the output file name

#   Format: Quarantined/PkgName_PkgVersion.tgz

file="Quarantined/${pkgName}_${pkgVer}.tgz"

# Basic transfer requirements

resource="/${bucket}/${file}"

contentType="application/x-compressed-tar"

# Added bits

acl="authenticated-read"

metaData="Quarantined Package"

storageType="REDUCED_REDUNDANCY"

# Calculate the signature.

dateValue=$(date -R)

stringToSign="PUT\n\n${contentType}\n${dateValue}\nx-amz-acl:${acl}\nx-amz-meta-reason:${metaData}\nx-amz-storage-class:${storageType}\n${resource}"

signature=$(

    echo -en "${stringToSign}" |

    openssl sha1 -hmac "${aws_secret_access_key}" -binary |

    base64

)

#echo dateValue: ${dateValue} 

#echo stringToSign: ${stringToSign}

#echo signature: ${signature}

#exit 1

# PUT!

if [ 1 ] ; then

    tmpTar=$( mktemp )

    [ $verbose -gt 0 ] && set -x

    [ $verbose -gt 0 ] && echo "Compress $pkgName/$pkgVer to file $tmpTar"

    tar -czf $tmpTar -C "$pkgBase" "$pkgName/$pkgVer"

    [ $verbose -gt 0 ] && echo "Transfer $pkgName/$pkgVer to bucket $bucket"

    curlVerbose=-s

    [ $verbose -gt 0 ] && curlVerbose=--verbose

    curl $curlVerbose \

        -X PUT \

        -T $tmpTar \

         --insecure \

        -H "Host: ${bucket}.s3.amazonaws.com" \

        -H "Date: ${dateValue}" \

        -H "Content-Type: ${contentType}" \

        -H "Authorization: AWS ${aws_access_key_id}:${signature}" \

        -H "x-amz-acl: ${acl}" \

        -H "x-amz-meta-reason: ${metaData}" \

        -H "x-amz-storage-class: ${storageType}" \

        "https://${bucket}.s3.amazonaws.com/${file}"

   set +x

fi

[ $verbose -gt 0 ] && echo "Transfer complete"

rm -rf $tmpTar

#############################################################

#   Fetch file info, just to be sure that the file got there

#   Get data about the file

#

#   Have seen that some files (large) do not appear immediately

#   Solution: Try a few times

#

# Calculate the HEAD signature.

#   Note the need for a triple \n

#   Is that because there is no contentType ?

#

dateValue=$(date -R)

stringToSign="HEAD\n\n\n${dateValue}\n${resource}"

signature=$(

    echo -en "${stringToSign}" |

    openssl sha1 -hmac "${aws_secret_access_key}" -binary |

    base64

)

#set -x

fileTest=0

for ii in $( seq 1 10 ); do 

    [ $verbose -gt 0 ] && echo "Testing file presence ($ii): ${file}"

    results=$(curl -I -X HEAD \

            -s \

             --insecure \

            -H "Host: ${bucket}.s3.amazonaws.com" \

            -H "Date: ${dateValue}" \

            -H "Authorization: AWS ${aws_access_key_id}:${signature}" \

            "https://${bucket}.s3.amazonaws.com/${file}" \

            )

    [ $verbose -gt 0 ] && echo "Testing file results: ${results}"

    if [[ "$results" =~ "HTTP/1.1 200 OK" ]]; then

        fileTest=1

        break

    fi

    echo "Test Fail: ${file}. Attempt: ${ii}"

    [ $verbose -gt 0 ] && echo "Test Failure. Wait and try again"

    sleep 5 

done

# Display results

if [ $fileTest -gt 0 ]; then

    if [ $verbose -gt 0 ] ; then

        echo "${ProgName}: Transferred:$pkgName/$pkgVer" 

    fi

    exit 0

else

    echo "${ProgName}: Error cannot access $pkgName/$pkgVer in S3 bucket ${bucket}"

    exit 1

fi