Skip to content

Instantly share code, notes, and snippets.

@jhgaylor

jhgaylor/meteor_download.js

Last active January 18, 2018 10:31
Show Gist options
  • Save jhgaylor/8345292 to your computer and use it in GitHub Desktop.
Save jhgaylor/8345292 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. jhgaylor revised this gist Jan 10, 2014. 1 changed file with 3 additions and 6 deletions.
    9 changes: 3 additions & 6 deletions meteor_download.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,10 +1,7 @@
    //given the urls of files to download, store them on the filesystem
    function download_all_gnip_files (urls, base_destination, job_id, cb) {
    function download_all_files (urls, base_destination, job_id, cb) {
    var url = urls.shift();
    // gather the path from the url and throw away the query parameters
    var path_fragment = url.split('_'+job_id)[1].split("?")[0];
    // the path to store the file
    var file_path = path.join(base_destination, job_id, path_fragment);
    var file_path = path.join(base_destination, job_id);

    // the path to the file without the filename
    var path_to_file_folder = path.dirname(file_path);
    @@ -18,7 +15,7 @@
    file.on('finish', function() {
    file.close();
    if(urls.length > 0){
    download_all_gnip_files(urls, base_destination, job_id, cb);
    download_all_files(urls, base_destination, job_id, cb);
    } else {
    cb();
    }
  2. jhgaylor created this gist Jan 10, 2014.
    37 changes: 37 additions & 0 deletions meteor_download.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,37 @@
    //given the urls of files to download, store them on the filesystem
    function download_all_gnip_files (urls, base_destination, job_id, cb) {
    var url = urls.shift();
    // gather the path from the url and throw away the query parameters
    var path_fragment = url.split('_'+job_id)[1].split("?")[0];
    // the path to store the file
    var file_path = path.join(base_destination, job_id, path_fragment);

    // the path to the file without the filename
    var path_to_file_folder = path.dirname(file_path);

    // the method to store a downloaded file to the fs
    // makes an http request and writes the response to a file
    function download_url_to_fs () {
    var file = fs.createWriteStream(file_path);
    var request = https.get(url, function(response) {
    response.pipe(file);
    file.on('finish', function() {
    file.close();
    if(urls.length > 0){
    download_all_gnip_files(urls, base_destination, job_id, cb);
    } else {
    cb();
    }
    });
    });
    }

    // bind download_url_to_fs to a meteor fiber
    var bound_download_url_to_fs = Meteor.bindEnvironment(download_url_to_fs, function (e) {
    throw e;
    });

    // verify the required paths exist or create it
    // and then download the file to from http to the js
    mkdirp(path_to_file_folder, 0777, bound_download_url_to_fs);
    }