Skip to content

Instantly share code, notes, and snippets.

@doktoil-makresh

doktoil-makresh/s3-multipart-upload.sh

Forked from jessewang-arvatosystems/s3-multipart-upload.sh
Last active September 11, 2019 17:47
Show Gist options
  • Save doktoil-makresh/acc7f4172c342b8a3c65a688535d12a7 to your computer and use it in GitHub Desktop.
Save doktoil-makresh/acc7f4172c342b8a3c65a688535d12a7 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. doktoil-makresh revised this gist Sep 11, 2019. 1 changed file with 51 additions and 12 deletions.
    63 changes: 51 additions & 12 deletions s3-multipart-upload.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -13,30 +13,69 @@
    # bash s3-multipart-upload.sh YOUR_FILE YOUR_BUCKET (OPTIONAL: PROFILE)
    # bash s3-multipart-upload.sh files.zip my-files
    # bash s3-multipart-upload.sh files.zip my-files second-profile
    if [ -z $1 ] ; then
    echo "You must to enter a filename as first argument"
    exit 2
    elif [ $1 == "help" ] ; then
    echo "Usage: s3-multipart-upload.sh filename bucket storage-class"
    else
    fileName=$1
    fi
    #Select a bucket
    if [ -z $2 ] ; then
    echo "Choose one bucket in this list:"
    aws s3 ls | awk '{print $3}'
    read bucket
    else
    bucket=$2
    fi

    fileName=$1
    bucket=$2
    profile=${3-default}
    #Choose a class-storage (could be one of:
    # STANDARD
    # REDUCED_REDUNDANCY
    # STANDARD_IA
    # ONEZONE_IA
    # INTELLIGENT_TIERING
    # GLACIER
    # DEEP_ARCHIVE
    # Technical details about class storages here: https://docs.aws.amazon.com/fr_fr/AmazonS3/latest/dev/storage-class-intro.html
    # Pricing details here: https://aws.amazon.com/fr/s3/pricing/
    if [ -z $3 ] ; then
    echo "We have to choose a class storage:
    STANDARD
    REDUCED_REDUNDANCY
    STANDARD_IA
    ONEZONE_IA
    INTELLIGENT_TIERING
    GLACIER
    DEEP_ARCHIVE
    Technical details about class storages here: https://docs.aws.amazon.com/fr_fr/AmazonS3/latest/dev/storage-class-intro.html
    Pricing details here: https://aws.amazon.com/fr/s3/pricing/"
    exit 2
    else
    classStorage=$3
    fi

    #Set to 90 MBs as default, 100 MBs is the limit for AWS files
    #Define temporary directory
    TMP_DIR=/tmp
    #Set to 90 MBs as default, 5000 MBs is the limit for AWS files
    mbSplitSize=90
    ((partSize = $mbSplitSize * 1000000))

    # Get main file size
    echo "Preparing $fileName for multipart upload"
    fileSize=`wc -c $fileName | awk '{print $1}'`
    fileSize=$(wc -c $fileName | awk '{print $1}')
    ((parts = ($fileSize+$partSize-1) / partSize))

    # Get main file hash
    mainMd5Hash=`openssl md5 -binary $fileName | base64`
    mainMd5Hash=$(openssl md5 -binary $fileName | base64)

    # Make directory to store temporary parts
    echo "Splitting $fileName into $parts temporary parts"
    mkdir temp-parts
    cp $fileName temp-parts/
    cd temp-parts

    mkdir ${TEMP_DIR}/temp-parts
    cd ${TEMP_DIR}temp-parts
    split -b $partSize $fileName
    rm $fileName

    # Create mutlipart upload
    echo "Initiating multipart upload for $fileName"
    @@ -48,8 +87,8 @@ for file in *
    do
    ((index++))
    echo "Uploading part $index of $parts..."
    hashData=`openssl md5 -binary $file | base64`
    eTag=`aws s3api upload-part --bucket $bucket --key $fileName --part-number $index --body $file --upload-id $uploadId --profile $profile | jq -r '.ETag'`
    hashData=$(openssl md5 -binary $file | base64)
    eTag=$(aws s3api upload-part --bucket $bucket --key $fileName --part-number $index --body $file --upload-id $uploadId --profile $profile | jq -r '.ETag')
    jsonData+="{\"ETag\":$eTag,\"PartNumber\":$index}"

    if (( $index == $parts ))
  2. @jessewang-arvatosystems jessewang-arvatosystems revised this gist Aug 24, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion s3-multipart-upload.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -12,7 +12,7 @@
    # Usage:
    # bash s3-multipart-upload.sh YOUR_FILE YOUR_BUCKET (OPTIONAL: PROFILE)
    # bash s3-multipart-upload.sh files.zip my-files
    # bash s3-multipart-upload.sh files.zip my-files second-proifle
    # bash s3-multipart-upload.sh files.zip my-files second-profile

    fileName=$1
    bucket=$2
  3. @jessewang-arvatosystems jessewang-arvatosystems revised this gist Aug 23, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion s3-multipart-upload.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -66,7 +66,7 @@ mainEtag=`aws s3api complete-multipart-upload --multipart-upload file://filepart
    if [[ $mainEtag != "" ]];
    then
    echo "Successfully uploaded: $fileName to S3 bucket: $bucket"
    else
    else
    echo "Something went wrong! $fileName was not uploaded to S3 bucket: $bucket"
    fi

  4. @jessewang-arvatosystems jessewang-arvatosystems revised this gist Aug 23, 2017. 1 changed file with 16 additions and 17 deletions.
    33 changes: 16 additions & 17 deletions s3-multipart-upload.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -45,30 +45,29 @@ uploadId=`aws s3api create-multipart-upload --bucket $bucket --key $fileName --m
    # Generate fileparts.json file that will be used at the end of the multipart upload
    jsonData="{\"Parts\":["
    for file in *
    do
    ((index++))
    echo "Uploading part $index of $parts..."
    hashData=`openssl md5 -binary $file | base64`
    do
    ((index++))
    echo "Uploading part $index of $parts..."
    hashData=`openssl md5 -binary $file | base64`
    eTag=`aws s3api upload-part --bucket $bucket --key $fileName --part-number $index --body $file --upload-id $uploadId --profile $profile | jq -r '.ETag'`
    jsonData+="{\"ETag\":$eTag,\"PartNumber\":$index}"

    eTag=`aws s3api upload-part --bucket $bucket --key $fileName --part-number $index --body $file --upload-id $uploadId --profile $profile | jq -r '.ETag'`
    jsonData+="{\"ETag\":$eTag,\"PartNumber\":$index}"

    if (( $index == $parts ))
    then
    jsonData+="]}"
    else
    jsonData+=","
    fi
    if (( $index == $parts ))
    then
    jsonData+="]}"
    else
    jsonData+=","
    fi
    done
    jq -n $jsonData > fileparts.json

    # Complete multipart upload, check ETag to verify success
    mainEtag=`aws s3api complete-multipart-upload --multipart-upload file://fileparts.json --bucket $bucket --key $fileName --upload-id $uploadId --profile $profile | jq -r '.ETag'`
    if [[ $mainEtag != "" ]];
    then
    echo "Successfully uploaded: $fileName to S3 bucket: $bucket"
    else
    echo "Something went wrong! $fileName was not uploaded to S3 bucket: $bucket"
    then
    echo "Successfully uploaded: $fileName to S3 bucket: $bucket"
    else
    echo "Something went wrong! $fileName was not uploaded to S3 bucket: $bucket"
    fi

    # Clean up files
  5. @jessewang-arvatosystems jessewang-arvatosystems revised this gist Aug 23, 2017. No changes.
  6. @jessewang-arvatosystems jessewang-arvatosystems revised this gist Aug 23, 2017. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion s3-multipart-upload.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -62,7 +62,7 @@ for file in *
    done
    jq -n $jsonData > fileparts.json

    # Complete multipart upload, check Etag to verify success
    # Complete multipart upload, check ETag to verify success
    mainEtag=`aws s3api complete-multipart-upload --multipart-upload file://fileparts.json --bucket $bucket --key $fileName --upload-id $uploadId --profile $profile | jq -r '.ETag'`
    if [[ $mainEtag != "" ]];
    then
  7. @jessewang-arvatosystems jessewang-arvatosystems created this gist Aug 23, 2017.
    76 changes: 76 additions & 0 deletions s3-multipart-upload.sh
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,76 @@
    #!/usr/bin/env bash

    # Copyright 2017 Jesse Wang
    # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
    # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    # This script requires:
    # - AWS CLI to be properly configured (https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html)
    # - Account has s3:PutObject access for the target S3 bucket

    # Usage:
    # bash s3-multipart-upload.sh YOUR_FILE YOUR_BUCKET (OPTIONAL: PROFILE)
    # bash s3-multipart-upload.sh files.zip my-files
    # bash s3-multipart-upload.sh files.zip my-files second-proifle

    fileName=$1
    bucket=$2
    profile=${3-default}

    #Set to 90 MBs as default, 100 MBs is the limit for AWS files
    mbSplitSize=90
    ((partSize = $mbSplitSize * 1000000))

    # Get main file size
    echo "Preparing $fileName for multipart upload"
    fileSize=`wc -c $fileName | awk '{print $1}'`
    ((parts = ($fileSize+$partSize-1) / partSize))

    # Get main file hash
    mainMd5Hash=`openssl md5 -binary $fileName | base64`

    # Make directory to store temporary parts
    echo "Splitting $fileName into $parts temporary parts"
    mkdir temp-parts
    cp $fileName temp-parts/
    cd temp-parts
    split -b $partSize $fileName
    rm $fileName

    # Create mutlipart upload
    echo "Initiating multipart upload for $fileName"
    uploadId=`aws s3api create-multipart-upload --bucket $bucket --key $fileName --metadata md5=$mainMd5Hash --profile $profile | jq -r '.UploadId'`

    # Generate fileparts.json file that will be used at the end of the multipart upload
    jsonData="{\"Parts\":["
    for file in *
    do
    ((index++))
    echo "Uploading part $index of $parts..."
    hashData=`openssl md5 -binary $file | base64`

    eTag=`aws s3api upload-part --bucket $bucket --key $fileName --part-number $index --body $file --upload-id $uploadId --profile $profile | jq -r '.ETag'`
    jsonData+="{\"ETag\":$eTag,\"PartNumber\":$index}"

    if (( $index == $parts ))
    then
    jsonData+="]}"
    else
    jsonData+=","
    fi
    done
    jq -n $jsonData > fileparts.json

    # Complete multipart upload, check Etag to verify success
    mainEtag=`aws s3api complete-multipart-upload --multipart-upload file://fileparts.json --bucket $bucket --key $fileName --upload-id $uploadId --profile $profile | jq -r '.ETag'`
    if [[ $mainEtag != "" ]];
    then
    echo "Successfully uploaded: $fileName to S3 bucket: $bucket"
    else
    echo "Something went wrong! $fileName was not uploaded to S3 bucket: $bucket"
    fi

    # Clean up files
    cd ..
    rm -R temp-parts