Skip to content

Instantly share code, notes, and snippets.

@freewayz

freewayz/aws-boto-s3-download-directory.py

Created October 6, 2016 15:35
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save freewayz/38f850a7b0c49f8e31c951c2eaf2c2c1 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save freewayz/38f850a7b0c49f8e31c951c2eaf2c2c1 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. freewayz created this gist Oct 6, 2016.
    47 changes: 47 additions & 0 deletions aws-boto-s3-download-directory.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,47 @@
    #!/usr/bin/env python

    import boto
    import sys, os
    from boto.s3.key import Key
    from boto.exception import S3ResponseError


    DOWNLOAD_LOCATION_PATH = os.path.expanduser("~") + "/s3-backup/"
    if not os.path.exists(DOWNLOAD_LOCATION_PATH):
    print ("Making download directory")
    os.mkdir(DOWNLOAD_LOCATION_PATH)


    def backup_s3_folder():
    BUCKET_NAME = "skoolsresources.com"
    AWS_ACCESS_KEY_ID= os.getenv("AWS_KEY_ID") # set your AWS_KEY_ID on your environment path
    AWS_ACCESS_SECRET_KEY = os.getenv("AWS_ACCESS_KEY") # set your AWS_ACCESS_KEY on your environment path
    conn = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_ACCESS_SECRET_KEY)
    bucket = conn.get_bucket(BUCKET_NAME)

    #goto through the list of files
    bucket_list = bucket.list()

    for l in bucket_list:
    key_string = str(l.key)
    s3_path = DOWNLOAD_LOCATION_PATH + key_string
    try:
    print ("Current File is ", s3_path)
    l.get_contents_to_filename(s3_path)
    except (OSError,S3ResponseError) as e:
    pass
    # check if the file has been downloaded locally
    if not os.path.exists(s3_path):
    try:
    os.makedirs(s3_path)
    except OSError as exc:
    # let guard againts race conditions
    import errno
    if exc.errno != errno.EEXIST:
    raise




    if __name__ == '__main__':
    backup_s3_folder()