max-mapper · February 14, 2017 21:54 · Feb 14, 2017 · Feb 14, 2017
diff --git a/readme.md b/readme.md
@@ -1,7 +1,10 @@
 Executive Office of the President Open Data Archive Backup
+
 Downloaded from https://catalog.data.gov/api/3/action/package_search?q=organization:eop-gov on Jan 20, 2017
 `hashes.txt` has the sha256 hash of the downloaded files from the URLs in the `result.results.*.resources` entries from the metadata linked above.
+
 A copy of the metadata from data.gov for the eop-gov collection is included in `meta.json`, and the download script I used is at `dl.js`
+
 Files missing at time of download are in `missing.txt`. Note many can be found at https://obamawhitehouse.archives.gov 
 
 Known Mirrors:

diff --git a/datagovmetadata.json b/datagovmetadata.json
diff --git a/dl.js b/dl.js
@@ -0,0 +1,25 @@
+var crypto = require('crypto')
+var request = require('request')
+var ndjson = require('ndjson')
+var parallel = require('parallel-transform')
+var fs = require('fs')
+var mkdirp = require('mkdirp')
+var pump = require('pump')
+var path = require('path')
+
+process.stdin
+  .pipe(ndjson.parse())
+  .pipe(parallel(5, dl))
+
+function dl (item, cb) {
+  var hash = crypto.createHash('sha256').update(item).digest('hex')
+  mkdirp('./' + hash, function () {
+    var req = request(item)  
+    req.on('error', function (err) {
+      console.log('Error', item, err.message)
+    })
+    var target = path.basename(item).split('?')[0]
+    var ws = fs.createWriteStream('./' + hash + '/' + target)
+    pump(req, ws, cb)
+  })
+}
diff --git a/hashes.txt b/hashes.txt
@@ -0,0 +1,125 @@
+007497d62cb97df9370c074f9e7e7f66a96b71ace9379958229c1ba7d166c177
+042bab1cd1bfe9f162c2321c61d1502de473ef4111f61309934427e24d20e2ea
+04babe76c0432651818d2c6d56e9bc63316696305ca18b432c27a5fe9911ff21
+0b4e611e906600725070ab1e3fed9bb0e1959d0e240329abcad91a85afff83f7
+11e2c5de94e88b8bcc806968f659ef8470a61d6d557b7828ba4e6432169ce65b
+140dfe6e821abe0b85a038db4395e1c827b71fdba74010d63b8b24c63a39d893
+154166ec27f8b04480748694599f2e10f4a68418ed8a77a2c6da443f59f5a230
+1795ca2278b8de1bad72cdd6a0addccf938c29a264df273e2986448111699933
+196668c9e7395f26b1be122b7fb306d9c783fd8a797673db1854ad080c6c6f60
+198b5fcf5411ca15e2c2e1b0a6c1d3e19e34f7bed7c411b83a3c47719f7384db
+1be25e6f200a07ceac2853bca358a5d13c2178192bc6e9bb449d6c15dec0ad90
+1d53a814942c5dacfc1dcedc85a2b42cac2308264f81523e96151b6c576ca01a
+1e213adc08dea4829bf13eb90f19900eea6ab5e4d9d2601d14771b54d0fa521a
+1ebe002278ae7b99b1b3a6d336bc45d00cf48be416148863aa52db805f707a30
+1fde7ee20fe5fbd2aad140ded008eac5b058c8739874ef842cc0950aec09a42c
+25c2446aaae57daa5a75efe8e195964c5208e70bad19fb9e4b6e821e094070c2
+2686678bb92d13993573e3f9c7ee9bc2b6a7d98d8a8d0579db8053fc5f3be678
+270c41405ec580a1d4e84abc3397f8a7f1ee6e23b7f35b19e818865e5e8191e5
+288105197ad48ee4931b97d6d78967b3d62f5cbd7d449e23fea40393c7f40e7d
+29a750e4c57831a95d0f61cc4a517368b798f77055920f8cb4e7acdbd1d9fd4e
+2b916a8e342a10f4cebbd1f83c58e18134e3f7ae21e32c769d680ffd8550715b
+2cee5097fd80181fb9c7baf6951858f92fd34975322ec1b62d12480eb34a08f8
+2fc22985b43bfc475b72769dfa1e1c9b30d085a35af1314b7a8f199aba911a28
+300045a5005e8f5a0fd64402dff1168f92fd7a3b22a492651e5d3fd0647b640b
+31c4008606a7afe076ac6829e9a0b5af9206a4f39e0e311336984d90305974b1
+3409a25d25c46f4bc0357fe3245f74545bbcd9eb438b5d5915445429d3a59e16
+371e2ff600640111de7bb0196ee3222c8cfa8d6803f549b266d71510ff3c1a04
+37817751e8779dad2ea3e3d7747c1a438c1fa66a8adb027d5593571ef5a3d3df
+3a3667b35631f61419122c2609275aa440856417bd28d67717f794fb10b3d2d8
+3dd9d29d56e9fd045ab49831d840b6828d1307241acd9ce310f25111e8338eec
+431fb5126bc0eb7a8573cc6749f5926310156bf442b101d5820426d6de4ee3eb
+48fbaf891898f605ae7d43bf26847a267cec16de6f37126180f3cc20b50dd311
+490cd37ba09293ab2fc38ab281d41d2f1439601bfd968a82f2016d7a90248ac2
+49dc8df1444255911100cbff848cd959ba08f97bfbc0252d8bf21ef7cd61426b
+4bfb12344ed1b0049e38a740cce3cfe427695901ef2f7bdf5602744e441759d6
+4cb74d3ff9b82101029dddee5363eb086022af20215542a0dd552d09af51b6ea
+4d3b885586e6192ff1f47d1cb58e0ffe881008e9fb7d3ec5b05710ffcfba6630
+4fb4febfee74039f19dbbd053d90f638bbe54788ee3328716a4a1c54cc09772a
+5079ad7003bac1375cf55e73a1406cd58d5cf5e8e7e4f6c13f0dbd01a8b04880
+5287f27c32c550eb9ec8197e046079a970cdae8e0b0f77b2657c02e436f8bf44
+53fc9204646ef6916e7026a092fd9c124589c10afd86bbc9fe10109d69db11d4
+54bd7aaf05d3a8224d0e365d2fc7ea65cb842d131bf0949c384e0430bab268e9
+558829b17b5f85e8e1d1c7c9cdebbe5a44132593e1fe7ea338d1052c2c91af1b
+5739c37dc986a94f1a2dcf31959d062fa7f61ee56927b733f4c72edc599b5f97
+590dde2468283d0b8e17fa5ce8bdb10f00a8035a8a60ab26c81557a3a64c4006
+59315eb502a66e081b7752144890093f396e4d0d7b2fab40577be6cb89407cb9
+5a51549c2adcf49dfcc3393ff65f6bd60f739da37ecab50e1056e41c80a62dc2
+5a80f4a81e02b23baddda62c0d44fbaa2770c5ccafd53ef28affde873950433a
+5bdb49f267c12cc1622bf43bbd237bc955808331cd6cfd137e7be8e7bbb6b2de
+60b85a3fd15faef73591e41590eedbc32048d8ee2b7ce13e8c65afa884523876
+624979cce87f1ab588d483f57381c5ffbe4203cdad38ca8b813af6f66fd92eff
+62f8e33613e10a689d78af8a29db5f14ef20ed8627487233989529efbbb54116
+630158d700e9b4025fd8a0eea07b4dfa9ef8b3351243c9058cb7f0a46e544f01
+66252ab2eac990b0752ca2287c385223ac39847e98554fb68c557c9156b84375
+675d746c8114ea9a70434203271e7a5bce4aef7749dd46a2642f8318c9a36aec
+680723fddec1e268854aec01e5c62530581fe9dd8da6e30a918b61f3af2fc2f8
+6cfcb6c882d12ab7b475ef60638f0c4fd8591556a0edbe7767c5dee8845eb00e
+6d2f68a9cfe79a1006d9a671023609b71f1746faf143627cf4553ad0fbb146aa
+6eedd91132156e6befce662699d5ce4ff8c89f5e47bcce7743de2a8ecb887dac
+7086af65160da53e30d62de484912ff618e0d0e15e58badfdc311ae4fa635ca0
+754bbbf36e89d8b6e3b446c0c44cea8b59d38cc993ff9b1890cc32ba8651e6f8
+788624b924fef30a2b2b4ca56a2ebb83b9bda86cfa4667fc43778c4029cabdea
+78c3d869d5026f7e90ab7fe5f06ddf6af2e8672fe7b088874c39fee71e94a4ae
+7c7b0ee6f0c4c79c7aa57d6c1b417d29766c44422a18548aa1485134b9be7ba0
+80d36c019ac43b83497b145e1129b1357c950ad1d607e3e8d9bec41c908c824f
+8308b04ce5397d063f86a2d7ef0d541f514eaca3ccfaf0f31bcde65658a5a9f9
+852b4225314d34883bf30026913f354e6fc28cadef1c8a047155c7054cb097c9
+8621fb7f765dea3b09237d16b7464d75db16105e88955c997a923e71f0df383a
+869baf9da68e9f3d73d997eb67508f4dc091c05a9459afc34fa7fb28ab7acb64
+885983f3ca8a738a9a736da32194777164ad89fedfdfc8aab22efabd4facbd2b
+8d352470c85d30db98e16bcf33c70c77aad8e9e2ff3b4217107a3110d9a5f11d
+942ae512e342412bfab8630abbe404cc6cb3d31cf31cda78a0fcc0ad805f44a3
+94acaae2ae8e8e6ec8299440e46876640cdd8fab6d24a19a3fd354a4ca44796e
+96d44860e3155f3b897eb758ea5ef7f1a9c57870ac4f9a3f23a151eda9dadf7b
+98e17d2cbaf00a647e85c87b5ab24d9d6b01745afb1fcbc8ad3ec3e573f936e6
+99388f1ebdbdf260aced52d347ecbf4719a5d32784fb965c3f364680888a3ad0
+9cd87a1ac360e0c685b7e64762f4f41c91517ba95a26c621c20408b7f5edd96b
+9ee2224e75602d4dd9f79cfd7b48769f3b8215722ac364262e8c43724ad74f33
+a279d468becc3c3b5da3cf62357981e34a482bee0e56acb1af0db0f8704f04d5
+a2b8d35eecc19c5a518295ee5220e496b688f66752faddba504e723345c4e22a
+a35478f9d0f2b5ba3be0c6ed519fe52b63c3b160b41ea2efe80aa064e6ea3b58
+a413d5e56d020bde4e3d6edd7e0dcc037b235cff3ea585120bf7f5cbae1df1be
+ae6a7a926858cd2bb8963065f982aa5bca3b01723220075c082ffba58c030d90
+ae814ac827cbda16f0b179b0d0893aa92940d2c410fc6f5ad54eb26ad99897a3
+af3531bd926cbcb845a9e1c8a25e444dd8eb9d170ed51c4174a870accb1c2cb8
+aff6b6c6ae0fc239a97e158d5a35e623d4b1596b65a41b0af30e7bf03f04fbbf
+b1b31c1c55474fa76e16f60635823a2c7d5d696e8f478b55f1c84a97f58997a9
+b283da6fcb1fee72cb8663f170652bb197a24ea133f5dbb76aa41925ab9b89e1
+b3d069eb1b21f504c5e2c8af19b91fe54c1437e12a48a29f2659a7faef941280
+b469626f9040c7546ef8c1b00d73f5058c415215c66849a70d43b214408a2215
+b5e9ac671e8c31e39c7c1e0b2afee73452acc0b1c7e5c38e28e9fe96a230c5fc
+b8d37a8b11547b114c948e1f28ca51aed012c2f774a70a15703badcb38a0bd19
+bb85a313941240179b8e721d616aceb120e22fb0411735858de6ad975f2a64de
+bbcd67bf31c775abf58ca70b2199af79eddd084fecb39e40707099018c2e966f
+bcfa43aed3419a394847929c658e927a755a4057f9309b79fc9bdbbbdfa6d0ab
+bdde015b747432f73b4f5e74fa914910730c37d2cd749ec711571cf466f09e1c
+c1c0daa108045197c5f8f4c0fee03117db257163cccafe7007767f40bc796f9e
+c21a559e171a9b542d6bdaffe3967ab69d0f2d4ebe1b83c40b06c35a43eefb0c
+c30763fbcfa63300c27e5024be29b04e77f5abe845653a9e17165990672f1717
+ce03034f0c97e760db69be0fd840ac53c00535bc5fedb6ff88f18401e72d59ea
+cea33b66fd06e369bd85707074a772b06b9734df6e75ee2badab920bdb0d427a
+d24e29ff7dfc6dcdee106d2723257a95d7128c293c47961351ea094f8bfd52e2
+d623de785d74cdf06f9bbf901b84c2d4c0d2156a2443ac3b5d0f01f52ebce22e
+dad3f9fd4311eae025eb3808c186d1bf62673556698cceb394ca73d8441075bd
+deebb71c8c01175f497af9be4b65a6518a741bb29a9946abe979a387432dee19
+e18791647fa5465e92162fccce8afd2e85d2f263b987d4d530eae401a33e0cae
+e19867dceb5b669859b561a6ae5978268abb1bbdb5f0ceebed5cfda191022ec3
+e2d6483217ae3bca2f3829c19b831fca60ca2a0e5d1b62f9effcf2d0006f3171
+e3bd9632fcc39ea8831dfc9c857badaa567092f6f6194e7a222d4e45f7815017
+e403e3cd64fae4f35eec871bfe4fad13c1153b457c93b1b0f1a97c00c823ce48
+e6da8e03f582aaed5bd027481bbabcb048c3ce0d56b1e6511305f2c357287339
+e7736b408eda445e32945237fd806c101a2e2d99c5168ec85616dd0597097887
+eaa6690a3046fdf4c30283b2d95112035c8c68474f3d6dc169d5978d921aac1c
+eb50520348548eaf356d84b5c77710e156b7946a0a945d2e997655b26cd92445
+ec66101139a2d53741282c8785e15f2e929d5377776f02338aa193aa466ddeb5
+eeafd2274657237de3b46213efe2a0f9cc5c73b8d2c28bed432720df7ceaa44e
+ef5f212c36222140741b9bdbb4603f523a0ccd1f81bc4b10c14337ed54c8ced9
+f0f1599c7a289a3a4871579511fc5411211e4f64af3a36735ca494312beebac4
+f394f130e8f814307e21ae389a255c98c72dfc57fded146a62c983d6d268a0b0
+f4303cb09b47795749b95c35e65ab541bb4b74392854cc3bef9d972ab88da031
+f7ecbfcfa69a135d4e7776533acb39e42e7b3282caae2b51c3b2bd620519480d
+f7ef24b9a70d08e571f799a1384b1a12ad5e76af01bcca5a09122948b804d3da
+f89dbba223208f4e74efe0082badfe02f4e4b9637a100c9fa8c065111eb0910d
+fa3edf9b5565fa201f5e7503aefb5cd51eb42953ad85ef2163a18db4168949b9
+fd07d3009be07e155545c2dcbd118a28f711cab8e8c7c2021a928d3bfb938aec
diff --git a/missing.txt b/missing.txt
@@ -0,0 +1,19 @@
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/outlays.xls
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/2012_lrb.xls
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/teb2012.xls
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/outlays.csv
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/receipts.xls
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/receipts.csv
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/historicadministrationforecasts.xls
+https://www.whitehouse.gov/omb/place/datasets
+http://go.usa.gov/ypD
+http://www.whitehouse.gov/omb/asset.aspx?AssetId=2290
+http://www.whitehouse.gov/omb/asset.aspx?AssetId=2541
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/budauth.xls
+http://www.whitehouse.gov/sites/default/files/omb/budget/fy2012/assets/budauth.csv
+http://go.usa.gov/ypZ
+http://earmarks.omb.gov/earmarks-public/resources/downloads/2009-appropriation-earmark-extract.zip
+http://go.usa.gov/ypW
+http://go.usa.gov/yp2
+http://www.itdashboard.gov/data_feeds/public?extype=cioRating&columns%5B%5D=uniqueInvestmentIdentifier&columns%5B%5D=businessCaseID&columns%5B%5D=agencyCode&columns%5B%5D=agencyName&columns%5B%5D=investmentTitle&columns%5B%5D=ratedDate&columns%5B%5D=evaluationByAgencyCIO&columns%5B%5D=comments&columns%5B%5D=visibleOnITDashboard&columns%5B%5D=cioRatingID&columns%5B%5D=investmentEventID&columns%5B%5D=updatedTime&agency_id=0&inv=&pager=1000%2C0&CSV=CSV
+http://go.usa.gov/yp4
diff --git a/readme.md b/readme.md
@@ -0,0 +1,11 @@
+Executive Office of the President Open Data Archive Backup
+Downloaded from https://catalog.data.gov/api/3/action/package_search?q=organization:eop-gov on Jan 20, 2017
+`hashes.txt` has the sha256 hash of the downloaded files from the URLs in the `result.results.*.resources` entries from the metadata linked above.
+A copy of the metadata from data.gov for the eop-gov collection is included in `meta.json`, and the download script I used is at `dl.js`
+Files missing at time of download are in `missing.txt`. Note many can be found at https://obamawhitehouse.archives.gov 
+
+Known Mirrors:
+
+- This one (48 datasets)
+- https://open.obamawhitehouse.archives.gov/ (28 datasets)
+- http://web.archive.org/web/20170118204606/https://open.whitehouse.gov/browse (31 datasets)