Subversion Repositories DevTools

#!/bin/bash
#   This script will quarantine a package-vesrion into S3 storage
#   The target bucket is: auawsaddp001
#   
#   The package vesrion will be tar-zipped
#   without the dpkg_archive prefix
#
#   The resulatant tar-zip will be transferred to S3
#
#       Reduced redundancy is used
#
#       Credentials: s3_dpkg user 
#
#       Usage:  ./savePkg /PathTo/pkgName/pkgVersion
#
function doHelp {
cat <<endOfHelp
    Command: savePkgToS3

    This program will tar-zip a dpkg_archive Package/Version and store it
    into an Amazon S3 bucket in a subdirectory called 'Quarantine'

    Options
    -h, --help              - Display this message
    -v, --verbose           - Increase verbosity
    -b, --bucket=name       - Specifies the name of the target bucket
    -p, --path=PVPath       - Specifies the path to the Package-Version to save
    -k, --key=keyVar        - Name of the EnvVar that conains the AWS key
                              Default is AWSKEY
    -s, --secret=secretVar  - Name of the EnvVar that conains the AWS secret
                              Default is AWSSECRET
endOfHelp
}

#
#   Init defaults
#
ProgName=savePkgtoS3
awsKeyVar=AWSKEY
awsSecretVar=AWSSECRET
verbose=0

# Note that we use `"$@"' to let each command-line parameter expand to a 
# separate word. The quotes around `$@' are essential!
# We need TEMP as the `eval set --' would nuke the return value of getopt.
TEMP=$( getopt -n ${ProgName} -o vhb:p:k:s: --long verbose,help,bucket:,path:,key:,secret: -- "$@" )

if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi

# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"

while true ; do
        case "$1" in
                -h|--help)    doHelp; exit 0;;
                -v|--verbose) let verbose++; shift 1;;
                -b|--bucket)  bucket="$2" ; shift 2 ;;
                -p|--path)    dpkgPath="$2" ; shift 2 ;;
                -k|--key)     awsKeyVar="$2" ; shift 2 ;;
                -s|--secret)  awsSecretVar="$2" ; shift 2 ;;
                --) shift ; break ;;
                *) echo "Internal error!" ; exit 1 ;;
        esac
done

if [ $verbose -gt 1 ] ; then
    echo bucket      :$bucket
    echo dpkgPath    :$dpkgPath
    echo awsKeyVar   :$awsKeyVar
    echo awsSecretVar:$awsSecretVar

    echo "Remaining arguments:"
    for arg do echo '--> '"\`$arg'" ; done
fi

: ${bucket:?No Bucket Specified}
: ${dpkgPath:?No Package Version Path}
: ${awsKeyVar:?No AWS Key specified}
: ${awsSecretVar:?No AWS Secret specified}

#
#   The KEY and the Secret are passed via EnvVars
#   The name of the vars are passed on the command line
#
aws_access_key_id=${!awsKeyVar}
aws_secret_access_key=${!awsSecretVar}

: ${aws_access_key_id:?No AWS Key found}
: ${aws_secret_access_key:?No AWS Secret found}

if [ $verbose -gt 1 ] ; then
    echo aws_access_key_id:$aws_access_key_id
    echo aws_secret_access_key:$aws_secret_access_key
fi

#
#   Determine
#       dpkgPath    - Cleanup the user arg
#       pkgBase     - Base of the package
#       pkgName     - Package Name
#       pkgVer      - Package Version
#

dpkgPath=${dpkgPath%/}

pkgBase=${dpkgPath%/*}
pkgBase=${pkgBase%/*}

pkgVer=${dpkgPath##*/}

pkgName=${dpkgPath%/*}
pkgName=${pkgName##*/}

if [ $verbose -gt 1 ] ; then
    echo pkgBase:$pkgBase
    echo pkgName:pkgName
    echo pkgVer:$pkgVer
fi

if [ ! -d $dpkgPath ] ; then
    echo "Error: PVPath does not address a directory"
    exit 1
fi

#
#   Create the output file name
#   Format: Quarantined/PkgName_PkgVersion.tgz

file="Quarantined/${pkgName}_${pkgVer}.tgz"

# Basic transfer requirements
resource="/${bucket}/${file}"
contentType="application/x-compressed-tar"

# Added bits
acl="authenticated-read"
metaData="Quarantined Package"
storageType="REDUCED_REDUNDANCY"


# Calculate the signature.
dateValue=$(date -R)
stringToSign="PUT\n\n${contentType}\n${dateValue}\nx-amz-acl:${acl}\nx-amz-meta-reason:${metaData}\nx-amz-storage-class:${storageType}\n${resource}"
signature=$(
    echo -en "${stringToSign}" |
    openssl sha1 -hmac "${aws_secret_access_key}" -binary |
    base64
)

#echo dateValue: ${dateValue} 
#echo stringToSign: ${stringToSign}
#echo signature: ${signature}
#exit 1

# PUT!
if [ 1 ] ; then
    tmpTar=$( mktemp )
    [ $verbose -gt 0 ] && set -x
    [ $verbose -gt 0 ] && echo "Compress $pkgName/$pkgVer to file $tmpTar"
    tar -czf $tmpTar -C "$pkgBase" "$pkgName/$pkgVer"

    [ $verbose -gt 0 ] && echo "Transfer $pkgName/$pkgVer to bucket $bucket"
    curlVerbose=-s
    [ $verbose -gt 0 ] && curlVerbose=--verbose

    curl $curlVerbose \
        -X PUT \
        -T $tmpTar \
         --insecure \
        -H "Host: ${bucket}.s3.amazonaws.com" \
        -H "Date: ${dateValue}" \
        -H "Content-Type: ${contentType}" \
        -H "Authorization: AWS ${aws_access_key_id}:${signature}" \
        -H "x-amz-acl: ${acl}" \
        -H "x-amz-meta-reason: ${metaData}" \
        -H "x-amz-storage-class: ${storageType}" \
        "https://${bucket}.s3.amazonaws.com/${file}"
   set +x
fi
[ $verbose -gt 0 ] && echo "Transfer complete"
rm -rf $tmpTar

#############################################################
#   Fetch file info, just to be sure that the file got there
#   Get data about the file
#
#   Have seen that some files (large) do not appear immediately
#   Solution: Try a few times
#
# Calculate the HEAD signature.
#   Note the need for a triple \n
#   Is that because there is no contentType ?
#
dateValue=$(date -R)
stringToSign="HEAD\n\n\n${dateValue}\n${resource}"
signature=$(
    echo -en "${stringToSign}" |
    openssl sha1 -hmac "${aws_secret_access_key}" -binary |
    base64
)

#set -x
fileTest=0
for ii in $( seq 1 10 ); do 
    [ $verbose -gt 0 ] && echo "Testing file presence ($ii): ${file}"
    results=$(curl -I -X HEAD \
            -s \
             --insecure \
            -H "Host: ${bucket}.s3.amazonaws.com" \
            -H "Date: ${dateValue}" \
            -H "Authorization: AWS ${aws_access_key_id}:${signature}" \
            "https://${bucket}.s3.amazonaws.com/${file}" \
            )
    [ $verbose -gt 0 ] && echo "Testing file results: ${results}"
    if [[ "$results" =~ "HTTP/1.1 200 OK" ]]; then
        fileTest=1
        break
    fi
    echo "Test Fail: ${file}. Attempt: ${ii}"
    [ $verbose -gt 0 ] && echo "Test Failure. Wait and try again"
    sleep 5 
done

# Display results
if [ $fileTest -gt 0 ]; then
    if [ $verbose -gt 0 ] ; then
        echo "${ProgName}: Transferred:$pkgName/$pkgVer" 
    fi
    exit 0
else
    echo "${ProgName}: Error cannot access $pkgName/$pkgVer in S3 bucket ${bucket}"
    exit 1
fi