joshwand · June 28, 2024 14:24 · Feb 2, 2014 · Feb 2, 2014 · Apr 17, 2013 · Apr 17, 2013
diff --git a/tumblr-likes-downloader.rb b/tumblr-likes-downloader.rb
@@ -19,7 +19,7 @@
 
 previous_dled_ids = YAML.load(File.open(HISTORY_FILE)) rescue []
 
-directory = "/Users/josh/new/tumblr"
+directory = "tumblr-likes"
 
 client = Tumblr::Client.new
 likes = client.likes

diff --git a/tumblr-likes-downloader.rb b/tumblr-likes-downloader.rb
@@ -1,5 +1,8 @@
 require 'tumblr_client'
 require 'mechanize'
+require 'date'
+require 'yaml'
+require 'uri'
 
 Tumblr.configure do |config|
   config.consumer_key = "consumer_key"
@@ -8,11 +11,15 @@
   config.oauth_token_secret = "token_secret"
 end
 
-
 THREADPOOL_SIZE = 4
 SEGMENT_SIZE = 25
 
-directory = "tumblr-likes"
+HISTORY_FILE = File.join(File.dirname(__FILE__), "previous.yml")
+
+
+previous_dled_ids = YAML.load(File.open(HISTORY_FILE)) rescue []
+
+directory = "/Users/josh/new/tumblr"
 
 client = Tumblr::Client.new
 likes = client.likes
@@ -25,14 +32,20 @@
 (0...liked_count).each do |i|
   if i==0 or i%SEGMENT_SIZE==0
     p "getting #{SEGMENT_SIZE} more likes starting at #{i}: #{likes.count} thus far"
-     client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do |like|
-       likes << like
-     end
-  end
+    client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do |like|
+     likes << like if like['type'] == 'photo' and !previous_dled_ids.include?(like['id'])
+   end
+ end
 
 end
 
-puts "#{likes.count} likes!"
+if likes.empty? 
+  p "no new likes!"
+  exit 0
+end 
+
+puts "#{likes.count} new likes!"
+
 
 
 # some of this code comes from https://github.com/jamiew/tumblr-photo-downloader
@@ -41,41 +54,45 @@
 
 threads = []
 likes.each_slice(likes.count / THREADPOOL_SIZE ).each do |group|
-  
+
   threads << Thread.new {
     begin
       p "launching thread #{threads.size + 1}"
 
       group.each do |like|
-      
+
         i = 0
         like["photos"].each do |photo|
           url = photo["original_size"]["url"]
 
-          file = Mechanize.new.get(url)
           filename = "#{like["blog_name"]}-#{like["slug"]}-"
           filename += "#{i}-" if i > 0
-          filename += File.basename(file.uri.to_s.split('?')[0])
-
-
+          filename += File.basename(URI.parse(url).path.split('?')[0])
           if File.exists?("#{directory}/#{filename}")
             puts "Already have #{url}"
             already_had += 1
           else
-            puts "Saving photo #{url}"
-            file.save_as("#{directory}/#{filename}")
+            begin
+              puts "Saving photo #{url}"
+              file = Mechanize.new.get(url)
+              file.save_as("#{directory}/#{filename}")
+            rescue Mechanize::ResponseCodeError => e
+              puts "Error #{e.response_code} getting file for #{url}"
+            end
           end
-
+          i += 1  
+          previous_dled_ids << like['id']            
         end
-        i += 1
       end
-    rescue Mechanize::ResponseCodeError
-      puts "Error getting file, #{$!}"
+    rescue Exception => e
+      puts "unhandled exception:, #{$!}"
     end
-      p "closing thread"
-    }
+    p "closing thread"
+  }
 
 end
 
 threads.each{|t| t.join }
 
+YAML.dump(previous_dled_ids, File.open(HISTORY_FILE, "w"))
+
diff --git a/tumblr-likes-downloader.rb b/tumblr-likes-downloader.rb
@@ -24,7 +24,7 @@
 
 (0...liked_count).each do |i|
   if i==0 or i%SEGMENT_SIZE==0
-    p "getting 50 more likes starting at #{i}: #{likes.count} thus far"
+    p "getting #{SEGMENT_SIZE} more likes starting at #{i}: #{likes.count} thus far"
      client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do |like|
        likes << like
      end

diff --git a/tumblr-likes-downloader.rb b/tumblr-likes-downloader.rb
@@ -0,0 +1,81 @@
+require 'tumblr_client'
+require 'mechanize'
+
+Tumblr.configure do |config|
+  config.consumer_key = "consumer_key"
+  config.consumer_secret = "consumer_secret"
+  config.oauth_token = "oath_token"
+  config.oauth_token_secret = "token_secret"
+end
+
+
+THREADPOOL_SIZE = 4
+SEGMENT_SIZE = 25
+
+directory = "tumblr-likes"
+
+client = Tumblr::Client.new
+likes = client.likes
+liked_count = likes["liked_count"]
+
+puts liked_count
+
+likes = []
+
+(0...liked_count).each do |i|
+  if i==0 or i%SEGMENT_SIZE==0
+    p "getting 50 more likes starting at #{i}: #{likes.count} thus far"
+     client.likes({:limit => SEGMENT_SIZE, :offset => i})["liked_posts"].each do |like|
+       likes << like
+     end
+  end
+
+end
+
+puts "#{likes.count} likes!"
+
+
+# some of this code comes from https://github.com/jamiew/tumblr-photo-downloader
+
+already_had = 0
+
+threads = []
+likes.each_slice(likes.count / THREADPOOL_SIZE ).each do |group|
+
+  threads << Thread.new {
+    begin
+      p "launching thread #{threads.size + 1}"
+
+      group.each do |like|
+
+        i = 0
+        like["photos"].each do |photo|
+          url = photo["original_size"]["url"]
+
+          file = Mechanize.new.get(url)
+          filename = "#{like["blog_name"]}-#{like["slug"]}-"
+          filename += "#{i}-" if i > 0
+          filename += File.basename(file.uri.to_s.split('?')[0])
+
+
+          if File.exists?("#{directory}/#{filename}")
+            puts "Already have #{url}"
+            already_had += 1
+          else
+            puts "Saving photo #{url}"
+            file.save_as("#{directory}/#{filename}")
+          end
+
+        end
+        i += 1
+      end
+    rescue Mechanize::ResponseCodeError
+      puts "Error getting file, #{$!}"
+    end
+      p "closing thread"
+    }
+
+end
+
+threads.each{|t| t.join }
+