#!/bin/bash # Usage: # bash s3BackUp.sh YOUR_BACKUP_DIRECTORY BACKUP_NAME YOUR_BUCKET MAIL_FROM MAIL_TO (OPTIONAL: S3_FOLDER PROFILE) # bash s3BackUp.sh /var/www/webdisk/example.com/ example my_bucket info@example.com soporte@example.com backup default # Arguments: readonly BACKUP_PATH_NO_REMOVE=$1 readonly BACKUP_NAME=$2 readonly S3_BUCKET_NAME=$3 readonly MAIL_FROM=${4} readonly MAIL_TO=${5} readonly S3_FOLDER=${6-backup} readonly PROFILE=${7-default} # Default: readonly PREFIX=backup_ readonly DATE=`date +%d-%m-%Y` readonly BACKUP_FILE_NAME=${PREFIX}${BACKUP_NAME}_${DATE}.tgz readonly BACKUP_PATH_FILE=${HOME}/${BACKUP_FILE_NAME} readonly S3_BUCKET_BACKUP=s3://${S3_BUCKET_NAME}/${S3_FOLDER}/ readonly S3_OUTPUT_BACKUP=${S3_FOLDER}/${BACKUP_FILE_NAME} readonly TEMP_PARTS=${HOME}/temp-parts readonly TEMP_EXTRACT=${HOME}/temp-extract ##### function finish() { rm -r ${TEMP_PARTS} 2> /dev/null rm -r ${TEMP_EXTRACT} 2> /dev/null rm ${BACKUP_PATH_FILE} 2> /dev/null } trap finish EXIT ##### function s3-send-mail() { local from=$1 local to=$2 local subject=$3 local text=$4 aws ses send-email \ --from $from \ --destination "ToAddresses=${to}" \ --message "Subject={Data=${subject},Charset=utf8},Body={Text={Data=${text},Charset=utf8},Html={Data=${text},Charset=utf8}}" } ##### function s3-multipart-upload() { sudo apt-get install -y -qq jq local filePath=$1 # file to upload local bucket=$2 # name of S3 bucket local s3Folder=$3 # destination of the file in S3 local dirParts=$4 # local folder where you create the parts of the file to send local profile=${5-default} # configuration profile of aws-cli #Set to 90 MBs as default, 100 MBs is the limit for AWS files mbSplitSize=90 ((partSize = $mbSplitSize * 1000000)) # Get main file size echo "Preparing $filePath for multipart upload" fileSize=`wc -c $filePath | awk '{print $1}'` ((parts = ($fileSize+$partSize-1) / partSize)) # Get main file hash mainMd5Hash=`openssl md5 -binary $filePath | base64` # Make directory to store temporary parts echo "Splitting $filePath into $parts temporary parts" rm -r ${dirParts} mkdir -p ${dirParts} cd ${dirParts} split -b $partSize $filePath # Create mutlipart upload echo "Initiating multipart upload for $filePath" uploadId=`aws s3api create-multipart-upload --bucket $bucket --key $s3Folder --metadata md5=$mainMd5Hash --profile $profile | jq -r '.UploadId'` # Generate fileparts.json file that will be used at the end of the multipart upload jsonData="{\"Parts\":[" for file in * do ((index++)) echo "Uploading part $index of $parts..." hashData=`openssl md5 -binary $file | base64` eTag=`aws s3api upload-part --bucket $bucket --key $s3Folder --part-number $index --body $file --upload-id $uploadId --profile $profile | jq -r '.ETag'` jsonData+="{\"ETag\":$eTag,\"PartNumber\":$index}" if (( $index == $parts )) then jsonData+="]}" else jsonData+="," fi done jq -n $jsonData > fileparts.json # Complete multipart upload, check ETag to verify success mainEtag=`aws s3api complete-multipart-upload --multipart-upload file://fileparts.json --bucket $bucket --key $s3Folder --upload-id $uploadId --profile $profile | jq -r '.ETag'` if [[ $mainEtag != "" ]]; then echo "Successfully uploaded: $filePath to S3 bucket: $bucket" else echo "Something went wrong! $filePath was not uploaded to S3 bucket: $bucket" # SEND FAULT REPORT s3-send-mail MAIL_FROM MAIL_TO "ALERT BACKUP FAULT - ${BACKUP_NAME}" "Verify the sending of file parts to the AWS S3 service" exit 1 fi # Clean up files rm -r ${TEMP_PARTS} cd .. } ##### function main() { # Release file in local and remote destination rm ${BACKUP_PATH_FILE} 2> /dev/null # CREATE BACKUP ARCHIVE tar czvf ${BACKUP_PATH_FILE} ${BACKUP_PATH_NO_REMOVE} # VERIFY BACKING ARCHIVE rm -r ${TEMP_EXTRACT} mkdir -p ${TEMP_EXTRACT} tar xzf ${BACKUP_PATH_FILE} --directory ${TEMP_EXTRACT} > /dev/null if [ $? != 0 ]; then echo "File is corrupted ... $1" # SEND FAULT REPORT s3-send-mail MAIL_FROM MAIL_TO "ALERT BACKUP FAULT - ${BACKUP_NAME}" "Check backup compression" exit 1 fi rm -r ${TEMP_EXTRACT} # SEND NEW BACKUP TO S3 s3-multipart-upload ${BACKUP_PATH_FILE} ${S3_BUCKET_NAME} ${S3_OUTPUT_BACKUP} ${TEMP_PARTS} ${PROFILE} # DELETE OLD BACKUP IN S3 aws s3 ls ${S3_BUCKET_BACKUP} | while read -r line; do strCreateDate=`echo $line|awk {'print $1" "$2'}` if date -d "${strCreateDate}" >/dev/null 2>&1 then createDate=`date --date "$strCreateDate" +%s` olderThan=`date --date "7 days ago" +%s` if [[ $createDate -lt $olderThan ]] then filePath=`echo $line|awk {'print $4'}` if [[ $filePath != "" ]] then aws s3 rm ${S3_BUCKET_BACKUP}$filePath fi fi fi done; } ##### main