#!/usr/bin/env ruby require 'rubygems' require 'aws-sdk' class S3FolderUpload attr_reader :folder_path, :total_files, :s3_bucket attr_accessor :files # Initialize the upload class # # folder_path - path to the folder that you want to upload # bucket - The bucket you want to upload to # aws_key - Your key generated by AWS defaults to the environemt setting AWS_KEY_ID # aws_secret - The secret generated by AWS # # Examples # => uploader = S3FolderUpload.new("some_route/test_folder", 'your_bucket_name') # def initialize(folder_path, bucket, aws_key = ENV['AWS_KEY_ID'], aws_secret = ENV['AWS_SECRET']) AWS.config(access_key_id: aws_key, secret_access_key: aws_secret, region: 'us-west-2') @folder_path = folder_path @files = Dir.glob("#{folder_path}/**/*") @total_files = files.length @connection = AWS::S3.new @s3_bucket = @connection.buckets[bucket] end # public: Upload files from the folder to S3 # # thread_count - How many threads you want to use (defaults to 5) # # Examples # => uploader.upload!(20) # true # => uploader.upload! # true # # Returns true when finished the process def upload!(thread_count = 5) file_number = 0 mutex = Mutex.new threads = [] thread_count.times do |i| threads[i] = Thread.new { until files.empty? mutex.synchronize do file_number += 1 Thread.current["file_number"] = file_number end file = files.pop rescue nil next unless file # I had some more manipulation here figuring out the git sha # For the sake of the example, we'll leave it simple # path = file puts "[#{Thread.current["file_number"]}/#{total_files}] uploading..." data = File.open(file) if File.directory?(data) data.close next else obj = s3_bucket.objects[path] obj.write(data, { acl: :public_read }) data.close end end } end threads.each { |t| t.join } end end uploader = S3FolderUpload.new('test', 'miles-media-library', AWS_KEY, AWS_SECRET) uploader.upload!(1)