var crypto = require('crypto')
var request = require('request')
var ndjson = require('ndjson')
var parallel = require('parallel-transform')
var fs = require('fs')
var mkdirp = require('mkdirp')
var pump = require('pump')
var path = require('path')

process.stdin
  .pipe(ndjson.parse())
  .pipe(parallel(5, dl))

function dl (item, cb) {
  var hash = crypto.createHash('sha256').update(item).digest('hex')
  mkdirp('./' + hash, function () {
    var req = request(item)  
    req.on('error', function (err) {
      console.log('Error', item, err.message)
    })
    var target = path.basename(item).split('?')[0]
    var ws = fs.createWriteStream('./' + hash + '/' + target)
    pump(req, ws, cb)
  })
}