lisovskyvlad · February 29, 2016 15:52 · Apr 18, 2013 · Apr 18, 2013 · Apr 18, 2013 · Dec 16, 2012
diff --git a/download-url-to-file.rb b/download-url-to-file.rb
@@ -5,7 +5,7 @@
 # specified in an external file.
 #
 # Author: Tobias Preuss
-# Revision: 2013-04-18 15:47 +0100 UTC
+# Revision: 2013-04-18 16:26 +0100 UTC
 # License: Creative Commons Attribution-ShareAlike 3.0 Unported
 
 require 'net/http'
@@ -50,16 +50,21 @@ def http_download_uri(uri, filename)
   puts "Starting HTTP download for: " + uri.to_s
   http_object = Net::HTTP.new(uri.host, uri.port)
   http_object.use_ssl = true if uri.scheme == 'https'
-  http_object.start do |http|
-    request = Net::HTTP::Get.new uri.request_uri
-    http.read_timeout = 500
-    http.request request do |response|
-      open filename, 'w' do |io|
-        response.read_body do |chunk|
-          io.write chunk
+  begin
+    http_object.start do |http|
+      request = Net::HTTP::Get.new uri.request_uri
+      http.read_timeout = 500
+      http.request request do |response|
+        open filename, 'w' do |io|
+          response.read_body do |chunk|
+            io.write chunk
+          end
         end
       end
     end
+  rescue Exception => e
+    puts "=> Exception: '#{e}'. Skipping download."
+    return
   end
   puts "Stored download as " + filename + "."
 end
@@ -68,10 +73,15 @@ def ftp_download_uri(uri, filename)
   puts "Starting FTP download for: " + uri.to_s + "."
   dirname = File.dirname(uri.path)
   basename = File.basename(uri.path)
-  Net::FTP.open(uri.host) do |ftp|
-    ftp.login
-    ftp.chdir(dirname)
-    ftp.getbinaryfile(basename)
+  begin
+    Net::FTP.open(uri.host) do |ftp|
+      ftp.login
+      ftp.chdir(dirname)
+      ftp.getbinaryfile(basename)
+    end
+  rescue Exception => e
+    puts "=> Exception: '#{e}'. Skipping download."
+    return
   end
   puts "Stored download as " + filename + "."
 end

diff --git a/download-url-to-file.rb b/download-url-to-file.rb
@@ -5,7 +5,7 @@
 # specified in an external file.
 #
 # Author: Tobias Preuss
-# Revision: 2012-03-16 18:49 +0100 UTC
+# Revision: 2013-04-18 15:47 +0100 UTC
 # License: Creative Commons Attribution-ShareAlike 3.0 Unported
 
 require 'net/http'
@@ -25,6 +25,8 @@ def create_directory(dirname)
 def read_uris_from_file(file)
   uris = Array.new
   File.open(file).each do |line|
+    line = line.strip
+    next if line == nil || line.length == 0
     parts = line.split(' ')
     pair = Hash[ [:resource, :filename].zip(parts) ]
     uris.push(pair)

diff --git a/download-url-to-file → download-url-to-file.rb b/download-url-to-file → download-url-to-file.rb
diff --git a/download-url-to-file b/download-url-to-file
@@ -6,6 +6,7 @@
 #
 # Author: Tobias Preuss
 # Revision: 2012-03-16 18:49 +0100 UTC
+# License: Creative Commons Attribution-ShareAlike 3.0 Unported
 
 require 'net/http'
 require 'net/ftp'

diff --git a/download-url-to-file b/download-url-to-file
@@ -58,19 +58,6 @@ def http_download_uri(uri, filename)
       end
     end
   end
-  # Net::HTTP.start(uri.host, uri.port) do |http|
-  #   http.use_ssl = true
-  #   http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-  #   request = Net::HTTP::Get.new uri.request_uri
-  #   http.read_timeout = 500
-  #   http.request request do |response|
-  #     open filename, 'w' do |io|
-  #       response.read_body do |chunk|
-  #         io.write chunk
-  #       end
-  #     end
-  #   end
-  # end
   puts "Stored download as " + filename + "."
 end
 

diff --git a/download-url-to-file b/download-url-to-file
@@ -45,7 +45,9 @@ end
 
 def http_download_uri(uri, filename)
   puts "Starting HTTP download for: " + uri.to_s
-  Net::HTTP.start(uri.host, uri.port) do |http|
+  http_object = Net::HTTP.new(uri.host, uri.port)
+  http_object.use_ssl = true if uri.scheme == 'https'
+  http_object.start do |http|
     request = Net::HTTP::Get.new uri.request_uri
     http.read_timeout = 500
     http.request request do |response|
@@ -56,6 +58,19 @@ def http_download_uri(uri, filename)
       end
     end
   end
+  # Net::HTTP.start(uri.host, uri.port) do |http|
+  #   http.use_ssl = true
+  #   http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  #   request = Net::HTTP::Get.new uri.request_uri
+  #   http.read_timeout = 500
+  #   http.request request do |response|
+  #     open filename, 'w' do |io|
+  #       response.read_body do |chunk|
+  #         io.write chunk
+  #       end
+  #     end
+  #   end
+  # end
   puts "Stored download as " + filename + "."
 end
 

diff --git a/download-url-to-file b/download-url-to-file
@@ -5,7 +5,7 @@
 # specified in an external file.
 #
 # Author: Tobias Preuss
-# Revision: 2012-03-16 18:22 +0100 UTC
+# Revision: 2012-03-16 18:49 +0100 UTC
 
 require 'net/http'
 require 'net/ftp'
@@ -84,16 +84,34 @@ def download_resources(pairs)
 end
 
 
-# Expected file format in `sources_file`:
-# http://www.domain.com/file target_file
-# ftp://www.domain.com/file target_file
-sources_file = ARGV.first
-uris = read_uris_from_file(sources_file)
+def main
+  sources_file = ARGV.first
+  uris = read_uris_from_file(sources_file)
 
-target_dir_name = Date.today.strftime('%y%m%d')
-create_directory(target_dir_name)
-Dir.chdir(target_dir_name)
-puts "Changed directory: " + Dir.pwd
+  target_dir_name = Date.today.strftime('%y%m%d')
+  create_directory(target_dir_name)
+  Dir.chdir(target_dir_name)
+  puts "Changed directory: " + Dir.pwd
 
-download_resources(uris)
+  download_resources(uris)
+end
+
+
+if __FILE__ == $0
+  usage = <<-EOU
+
+usage: ruby #{File.basename($0)} sources.txt
+
+  The file sources.txt should contain an URL and the target file name.
+  The expected file format is:
 
+  http://www.domain.com/file target_file_name
+  ftp://www.domain.com/file target_file_name
+
+  EOU
+
+  abort usage if ARGV.length != 1
+
+  main
+
+end
diff --git a/download-url-to-file b/download-url-to-file
@@ -1,4 +1,11 @@
 #!/usr/bin/env ruby
+#
+# Ruby script to download a number of files
+# from individual URLs via HTTP/HTTPS/FTP
+# specified in an external file.
+#
+# Author: Tobias Preuss
+# Revision: 2012-03-16 18:22 +0100 UTC
 
 require 'net/http'
 require 'net/ftp'
@@ -37,7 +44,7 @@ def download_resource(resource, filename)
 end
 
 def http_download_uri(uri, filename)
-  puts "Starting HTTP download for " + filename + "."
+  puts "Starting HTTP download for: " + uri.to_s
   Net::HTTP.start(uri.host, uri.port) do |http|
     request = Net::HTTP::Get.new uri.request_uri
     http.read_timeout = 500
@@ -49,18 +56,19 @@ def http_download_uri(uri, filename)
       end
     end
   end
-  puts "Finished download for " + filename + "."
+  puts "Stored download as " + filename + "."
 end
 
 def ftp_download_uri(uri, filename)
-  puts "Starting FTP download for " + filename + "."
+  puts "Starting FTP download for: " + uri.to_s + "."
   dirname = File.dirname(uri.path)
   basename = File.basename(uri.path)
   Net::FTP.open(uri.host) do |ftp|
     ftp.login
     ftp.chdir(dirname)
     ftp.getbinaryfile(basename)
   end
+  puts "Stored download as " + filename + "."
 end
 
 def download_resources(pairs)
@@ -76,7 +84,9 @@ def download_resources(pairs)
 end
 
 
-# Expected file format: http://www.domain.com/file sources_file
+# Expected file format in `sources_file`:
+# http://www.domain.com/file target_file
+# ftp://www.domain.com/file target_file
 sources_file = ARGV.first
 uris = read_uris_from_file(sources_file)
 

diff --git a/download-url-to-file b/download-url-to-file
@@ -1,21 +1,20 @@
 #!/usr/bin/env ruby
 
 require 'net/http'
+require 'net/ftp'
 require 'uri'
 require 'date'
 
-# Expected file format: http://www.domain.com/file filename
-file = ARGV.first
 
-def createDirectory(dirname)
+def create_directory(dirname)
   unless Dir.exists?(dirname)
     Dir.mkdir(dirname)
   else
     puts "Skipping creating directory " + dirname + ". It already exists."
   end
 end
 
-def readUris(file)
+def read_uris_from_file(file)
   uris = Array.new
   File.open(file).each do |line|
     parts = line.split(' ')
@@ -25,9 +24,20 @@ def readUris(file)
   uris
 end
 
-def download(resource, filename)
-  puts "Starting download for " + filename + "."
-  uri = URI(resource)
+def download_resource(resource, filename)
+  uri = URI.parse(resource)
+  case uri.scheme.downcase
+  when /http|https/
+    http_download_uri(uri, filename)
+  when /ftp/
+    ftp_download_uri(uri, filename)
+  else
+    puts "Unsupported URI scheme for resource " + resource + "."
+  end
+end
+
+def http_download_uri(uri, filename)
+  puts "Starting HTTP download for " + filename + "."
   Net::HTTP.start(uri.host, uri.port) do |http|
     request = Net::HTTP::Get.new uri.request_uri
     http.read_timeout = 500
@@ -42,21 +52,38 @@ def download(resource, filename)
   puts "Finished download for " + filename + "."
 end
 
-def downloadAll(pairs)
+def ftp_download_uri(uri, filename)
+  puts "Starting FTP download for " + filename + "."
+  dirname = File.dirname(uri.path)
+  basename = File.basename(uri.path)
+  Net::FTP.open(uri.host) do |ftp|
+    ftp.login
+    ftp.chdir(dirname)
+    ftp.getbinaryfile(basename)
+  end
+end
+
+def download_resources(pairs)
   pairs.each do |pair|
     filename = pair[:filename].to_s
     resource = pair[:resource].to_s
     unless File.exists?(filename)
-      download(resource, filename)
+      download_resource(resource, filename)
     else
       puts "Skipping download for " + filename + ". It already exists."
     end
   end
 end
 
 
-dirname = Date.today.strftime('%y%m%d')
-createDirectory(dirname)
-Dir.chdir(dirname)
-puts Dir.pwd
-downloadAll(readUris(file))
+# Expected file format: http://www.domain.com/file sources_file
+sources_file = ARGV.first
+uris = read_uris_from_file(sources_file)
+
+target_dir_name = Date.today.strftime('%y%m%d')
+create_directory(target_dir_name)
+Dir.chdir(target_dir_name)
+puts "Changed directory: " + Dir.pwd
+
+download_resources(uris)
+
diff --git a/download-url-to-file b/download-url-to-file
@@ -10,10 +10,8 @@ file = ARGV.first
 def createDirectory(dirname)
   unless Dir.exists?(dirname)
     Dir.mkdir(dirname)
-    return true
   else
     puts "Skipping creating directory " + dirname + ". It already exists."
-    return false
   end
 end
 
@@ -58,8 +56,7 @@ end
 
 
 dirname = Date.today.strftime('%y%m%d')
-if createDirectory(dirname)
-  Dir.chdir(dirname)
-  puts Dir.pwd
-  downloadAll(readUris(file))
-end
+createDirectory(dirname)
+Dir.chdir(dirname)
+puts Dir.pwd
+downloadAll(readUris(file))
diff --git a/download-url-to-file b/download-url-to-file
@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+require 'net/http'
+require 'uri'
+require 'date'
+
+# Expected file format: http://www.domain.com/file filename
+file = ARGV.first
+
+def createDirectory(dirname)
+  unless Dir.exists?(dirname)
+    Dir.mkdir(dirname)
+    return true
+  else
+    puts "Skipping creating directory " + dirname + ". It already exists."
+    return false
+  end
+end
+
+def readUris(file)
+  uris = Array.new
+  File.open(file).each do |line|
+    parts = line.split(' ')
+    pair = Hash[ [:resource, :filename].zip(parts) ]
+    uris.push(pair)
+  end
+  uris
+end
+
+def download(resource, filename)
+  puts "Starting download for " + filename + "."
+  uri = URI(resource)
+  Net::HTTP.start(uri.host, uri.port) do |http|
+    request = Net::HTTP::Get.new uri.request_uri
+    http.read_timeout = 500
+    http.request request do |response|
+      open filename, 'w' do |io|
+        response.read_body do |chunk|
+          io.write chunk
+        end
+      end
+    end
+  end
+  puts "Finished download for " + filename + "."
+end
+
+def downloadAll(pairs)
+  pairs.each do |pair|
+    filename = pair[:filename].to_s
+    resource = pair[:resource].to_s
+    unless File.exists?(filename)
+      download(resource, filename)
+    else
+      puts "Skipping download for " + filename + ". It already exists."
+    end
+  end
+end
+
+
+dirname = Date.today.strftime('%y%m%d')
+if createDirectory(dirname)
+  Dir.chdir(dirname)
+  puts Dir.pwd
+  downloadAll(readUris(file))
+end