-
-
Save lisovskyvlad/39f3984e9be866fe2b68 to your computer and use it in GitHub Desktop.
Ruby script to download a number of files from individual URLs via HTTP/HTTPS/FTP specified in an external file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env ruby | |
| require 'net/http' | |
| require 'net/ftp' | |
| require 'uri' | |
| require 'date' | |
| def create_directory(dirname) | |
| unless Dir.exists?(dirname) | |
| Dir.mkdir(dirname) | |
| else | |
| puts "Skipping creating directory " + dirname + ". It already exists." | |
| end | |
| end | |
| def read_uris_from_file(file) | |
| uris = Array.new | |
| File.open(file).each do |line| | |
| parts = line.split(' ') | |
| pair = Hash[ [:resource, :filename].zip(parts) ] | |
| uris.push(pair) | |
| end | |
| uris | |
| end | |
| def download_resource(resource, filename) | |
| uri = URI.parse(resource) | |
| case uri.scheme.downcase | |
| when /http|https/ | |
| http_download_uri(uri, filename) | |
| when /ftp/ | |
| ftp_download_uri(uri, filename) | |
| else | |
| puts "Unsupported URI scheme for resource " + resource + "." | |
| end | |
| end | |
| def http_download_uri(uri, filename) | |
| puts "Starting HTTP download for " + filename + "." | |
| Net::HTTP.start(uri.host, uri.port) do |http| | |
| request = Net::HTTP::Get.new uri.request_uri | |
| http.read_timeout = 500 | |
| http.request request do |response| | |
| open filename, 'w' do |io| | |
| response.read_body do |chunk| | |
| io.write chunk | |
| end | |
| end | |
| end | |
| end | |
| puts "Finished download for " + filename + "." | |
| end | |
| def ftp_download_uri(uri, filename) | |
| puts "Starting FTP download for " + filename + "." | |
| dirname = File.dirname(uri.path) | |
| basename = File.basename(uri.path) | |
| Net::FTP.open(uri.host) do |ftp| | |
| ftp.login | |
| ftp.chdir(dirname) | |
| ftp.getbinaryfile(basename) | |
| end | |
| end | |
| def download_resources(pairs) | |
| pairs.each do |pair| | |
| filename = pair[:filename].to_s | |
| resource = pair[:resource].to_s | |
| unless File.exists?(filename) | |
| download_resource(resource, filename) | |
| else | |
| puts "Skipping download for " + filename + ". It already exists." | |
| end | |
| end | |
| end | |
| # Expected file format: http://www.domain.com/file sources_file | |
| sources_file = ARGV.first | |
| uris = read_uris_from_file(sources_file) | |
| target_dir_name = Date.today.strftime('%y%m%d') | |
| create_directory(target_dir_name) | |
| Dir.chdir(target_dir_name) | |
| puts "Changed directory: " + Dir.pwd | |
| download_resources(uris) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment