Skip to content

Instantly share code, notes, and snippets.

@jerem

jerem/s3delete.py

Last active October 9, 2015 10:28
Show Gist options
  • Save jerem/3489032 to your computer and use it in GitHub Desktop.
Save jerem/3489032 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. jerem revised this gist Jun 7, 2013. 1 changed file with 13 additions and 7 deletions.
    20 changes: 13 additions & 7 deletions s3delete.py
    View file
    Original file line number Diff line number Diff line change
    @@ -15,14 +15,20 @@ def delete(access_key, access_secret, bucket_name, concurency):
    conn = S3Connection(access_key, access_secret)
    bucket = Bucket(conn, bucket_name)

    i = 0
    while True:
    keys = bucket.get_all_keys(max_keys=concurency)
    if len(keys) == 0:
    print('All keys deleted!')
    keys = bucket.get_all_keys()
    if not keys:
    break
    print(('Deleting {0} keys...'.format(len(keys))))
    threads = [gevent.spawn(bucket.delete_key, k) for k in keys]
    gevent.joinall(threads)
    subset = []
    for k in keys:
    subset.append(k)
    i += 1
    if len(subset) == concurency:
    print(('Deleting {0} keys... (total = {1})'.format(len(subset), i)))
    threads = [gevent.spawn(bucket.delete_key, s) for s in subset]
    gevent.joinall(threads)
    subset = []

    conn.delete_bucket(bucket)
    print('Bucket deleted!')
    @@ -45,7 +51,7 @@ def main(argv):
    '-c', '--concurrency',
    dest='concurrency', type='int',
    help='Number of keys to delete simultaneously',
    default=200
    default=20
    )

    options, args = parser.parse_args()
  2. jerem created this gist Aug 27, 2012.
    64 changes: 64 additions & 0 deletions s3delete.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,64 @@
    #!/usr/bin/env python

    import gevent.monkey
    gevent.monkey.patch_all()

    import sys
    import optparse
    import gevent

    from boto.s3.connection import S3Connection
    from boto.s3.bucket import Bucket


    def delete(access_key, access_secret, bucket_name, concurency):
    conn = S3Connection(access_key, access_secret)
    bucket = Bucket(conn, bucket_name)

    while True:
    keys = bucket.get_all_keys(max_keys=concurency)
    if len(keys) == 0:
    print('All keys deleted!')
    break
    print(('Deleting {0} keys...'.format(len(keys))))
    threads = [gevent.spawn(bucket.delete_key, k) for k in keys]
    gevent.joinall(threads)

    conn.delete_bucket(bucket)
    print('Bucket deleted!')


    def main(argv):
    parser = optparse.OptionParser()
    parser.set_usage('%prog [options] <bucket>')
    parser.add_option(
    '-a', '--access_key',
    dest='access_key', type='string',
    help='Your AWS Access Key ID'
    )
    parser.add_option(
    '-s', '--access_secret',
    dest='access_secret', type='string',
    help='Your AWS Secret Access Key'
    )
    parser.add_option(
    '-c', '--concurrency',
    dest='concurrency', type='int',
    help='Number of keys to delete simultaneously',
    default=200
    )

    options, args = parser.parse_args()
    if not args or len(args) != 1:
    parser.print_help()
    sys.exit(1)

    delete(
    options.access_key,
    options.access_secret,
    args[0],
    options.concurrency
    )

    if __name__ == '__main__':
    main(sys.argv)