Skip to content

Instantly share code, notes, and snippets.

@snoj

snoj/download.ps1

Created March 24, 2017 14:00
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save snoj/9a1f3cb85ae76c75ce85f13fb9ba4204 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save snoj/9a1f3cb85ae76c75ce85f13fb9ba4204 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. snoj created this gist Mar 24, 2017.
    43 changes: 43 additions & 0 deletions download.ps1
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,43 @@
    $root = "http://pdf.quad.download.s3.amazonaws.com/";
    $mroot = $root;
    if($pdfs -eq $null -or $pdfs.count -eq 0) {
    $pdfs = @{};
    while(1) {
    $mroot;
    $res = [xml](iwr $mroot | select -ExpandProperty content);

    $res.ListBucketResult.Contents | ?{ [regex]::IsMatch($_.key, "\.pdf`$", "IgnoreCase") } | %{
    try {
    $pdfs.Add($_.key, $_);
    } catch { }
    }
    $marker = $res.ListBucketResult.Contents | select -Last 1

    $mroot = $root + "?marker=" + $marker.Key;

    if($res.ListBucketResult.Contents.Length -lt 1000) {
    break;
    }
    }
    }

    mkdir "C:\topologymaps\";

    $dsb = {
    param($root, $fn)
    if((Test-Path (Join-Path "C:\topologymaps\" $fn))) {
    return;
    }
    iwr ($root + $fn) -OutFile (Join-Path "C:\topologymaps\" $fn);
    };
    $djobs = @();
    $pdfs.Keys | %{

    Get-Job -State Completed | Remove-Job;

    if(($djobs | ?{ $_.State -eq "Running"}).Count -gt 4) {
    ($djobs | ?{ $_.State -eq "Running"}) | Wait-Job -Any
    }
    $tj = Start-Job -ScriptBlock $dsb -ArgumentList @($root, $_);
    $djobs +=($tj);
    }