JackDanger · October 26, 2010 04:10 · Oct 26, 2010 · Oct 26, 2010 · Oct 26, 2010
diff --git a/s3_bucket_clone.rb b/s3_bucket_clone.rb
@@ -1,17 +1,21 @@
-# Taken from: http://snippets.dzone.com/posts/show/4935
+#!/usr/bin/env ruby
+## Taken from: http://snippets.dzone.com/posts/show/4935
 #
 # Usage:
 # a = AmazonS3Asset.new
 # a.copy_over_bucket("myapp_production", "myapp_production")
 
 require 'aws/s3'
 require 'mechanize'
+require 'logger'
+require 'yaml'
 
 class AmazonS3Asset
-  
+
   include AWS::S3
-  S3ID = "your s3 id"
-  S3KEY = "your s3 key"
+  CONFIG = YAML.load_file File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 's3.yml'))
+  S3ID  = CONFIG['aws_access_key']
+  S3KEY = CONFIG['aws_secret_access_key']
 
   def initialize
     puts "connecting..."
@@ -28,29 +32,30 @@ def delete_key(bucket, key)
   end
 
   def empty_bucket(bucket)
-    bucket_keys(bucket).each do |k|
-      puts "deleting #{k}"
+    each_bucket_key(bucket) do |k|
+      puts "deleting #{bucket}: #{k}"
       delete_key(bucket,k)
     end
   end
 
-  def bucket_keys(bucket)
+  def each_bucket_key(bucket_name, &block)
     marker_str = ""
-    return_objs = []
+    bucket = Bucket.find(bucket_name)
     begin
-        b = Bucket.find(bucket, :marker => marker_str)
-        marker_str = b.objects.last.key unless b.empty?
-        return_objs.concat b.objects.collect { |o| o.key }
-    end while not objs.empty?
-    return return_objs
+      objects = bucket.objects(:marker => marker_str)
+      objects.each do |object|
+        yield object.key
+      end
+      marker_str = objects.last.key unless objects.empty?
+    end while not objects.empty?
   end
 
   def copy_over_bucket(from_bucket, to_bucket)
     puts "Replacing #{to_bucket} with contents of #{from_bucket}"
-    #delete to_bucket
+    # delete to_bucket
     empty_bucket(to_bucket)
-    bucket_keys(from_bucket).each do |k|
-      copy_between_buckets(from_bucket, to_bucket, k)
+    each_bucket_key(from_bucket) do |key|
+      copy_between_buckets(from_bucket, to_bucket, key)
     end
   end
 
@@ -81,16 +86,18 @@ def store_file(bucket, key, filename)
 
   def download(url, save_as = nil)
     if save_as.nil?
-      Dir.mkdir("amazon_s3_temp") if !File.exists?("amazon_s3_temp")
-      save_as = File.join("amazon_s3_temp",File.basename(url))
+      tmp_dir = '/tmp/amazon_s3_temp'
+      Dir.mkdir(tmp_dir) unless File.exists?(tmp_dir)
+      save_as = File.join(tmp_dir, File.basename(url))
     end
     begin
       puts "Saving #{url} to #{save_as}"
-      agent = WWW::Mechanize.new {|a| a.log = Logger.new(STDERR) }
+      agent = Mechanize.new
       img = agent.get(url)
       img.save_as(save_as)
       return save_as
-    rescue
+    rescue => e
+      puts e
       raise "Failed on " + url + "  " + save_as
     end
   end
@@ -104,4 +111,4 @@ def exists?(bucket,key)
     return !res.nil?
   end
 
-end
+end
diff --git a/s3_bucket_clone.rb b/s3_bucket_clone.rb
@@ -1,7 +1,7 @@
 # Taken from: http://snippets.dzone.com/posts/show/4935
 #
 # Usage:
-# a = AmazoneS3Asset.new
+# a = AmazonS3Asset.new
 # a.copy_over_bucket("myapp_production", "myapp_production")
 
 require 'aws/s3'

diff --git a/s3_bucket_clone.rb b/s3_bucket_clone.rb
@@ -0,0 +1,107 @@
+# Taken from: http://snippets.dzone.com/posts/show/4935
+#
+# Usage:
+# a = AmazoneS3Asset.new
+# a.copy_over_bucket("myapp_production", "myapp_production")
+
+require 'aws/s3'
+require 'mechanize'
+
+class AmazonS3Asset
+
+  include AWS::S3
+  S3ID = "your s3 id"
+  S3KEY = "your s3 key"
+
+  def initialize
+    puts "connecting..."
+    AWS::S3::Base.establish_connection!(
+      :access_key_id     => S3ID,
+      :secret_access_key => S3KEY
+    )
+  end
+
+  def delete_key(bucket, key)
+    if exists?(bucket, key) 
+      S3Object.delete key, bucket
+    end
+  end
+
+  def empty_bucket(bucket)
+    bucket_keys(bucket).each do |k|
+      puts "deleting #{k}"
+      delete_key(bucket,k)
+    end
+  end
+
+  def bucket_keys(bucket)
+    marker_str = ""
+    return_objs = []
+    begin
+        b = Bucket.find(bucket, :marker => marker_str)
+        marker_str = b.objects.last.key unless b.empty?
+        return_objs.concat b.objects.collect { |o| o.key }
+    end while not objs.empty?
+    return return_objs
+  end
+
+  def copy_over_bucket(from_bucket, to_bucket)
+    puts "Replacing #{to_bucket} with contents of #{from_bucket}"
+    #delete to_bucket
+    empty_bucket(to_bucket)
+    bucket_keys(from_bucket).each do |k|
+      copy_between_buckets(from_bucket, to_bucket, k)
+    end
+  end
+
+  def copy_between_buckets(from_bucket, to_bucket, from_key, to_key = nil)
+    if exists?(from_bucket, from_key)
+      to_key = from_key if to_key.nil?
+      puts "Copying #{from_bucket}.#{from_key} to #{to_bucket}.#{to_key}"
+      url = "http://s3.amazonaws.com/#{from_bucket}/#{from_key}"
+      filename = download(url)
+      store_file(to_bucket,to_key,filename)
+      File.delete(filename)
+      return "http://s3.amazonaws.com/#{to_bucket}/#{to_key}"
+    else
+      puts "#{from_bucket}.#{from_key} didn't exist"
+      return nil
+    end
+  end
+
+  def store_file(bucket, key, filename)
+     puts "Storing #{filename} in #{bucket}.#{key}"
+     S3Object.store(
+      key,
+      File.open(filename),
+      bucket,
+      :access => :public_read
+      )
+  end
+
+  def download(url, save_as = nil)
+    if save_as.nil?
+      Dir.mkdir("amazon_s3_temp") if !File.exists?("amazon_s3_temp")
+      save_as = File.join("amazon_s3_temp",File.basename(url))
+    end
+    begin
+      puts "Saving #{url} to #{save_as}"
+      agent = WWW::Mechanize.new {|a| a.log = Logger.new(STDERR) }
+      img = agent.get(url)
+      img.save_as(save_as)
+      return save_as
+    rescue
+      raise "Failed on " + url + "  " + save_as
+    end
+  end
+
+  def exists?(bucket,key)
+    begin
+      res = S3Object.find key, bucket
+    rescue 
+      res = nil
+    end
+    return !res.nil?
+  end
+
+end
No results found