jjjake · October 29, 2015 18:01 · Oct 29, 2015
diff --git a/ia-mine.go b/ia-mine.go
@@ -0,0 +1,66 @@
+package main
+
+import (
+    "os"
+    "bufio"
+    "crypto/tls"
+    "net/http"
+    "io/ioutil"
+    "fmt"
+    "time"
+)
+
+type Resp struct {
+    *http.Response
+    err error
+}
+
+func fetch(uri string, queue chan string) {
+    tr := &http.Transport{
+        TLSClientConfig:    &tls.Config{},
+    }
+
+    client := http.Client{Transport: tr}
+    resp, err := client.Get(uri)
+    defer resp.Body.Close()
+
+    if err != nil {
+        // re-queue
+        queue <- uri
+        return
+    }
+
+    r := Resp{resp, err}
+    body, _ := ioutil.ReadAll(r.Body)
+
+    fmt.Println(string(body))
+}
+
+func q(itemlist string, queue chan string) {
+    file, _ := os.Open(itemlist)
+    defer file.Close()
+    scanner := bufio.NewScanner(file)
+
+    for scanner.Scan() {
+        uri := "http://archive.org/metadata/" + scanner.Text()
+        queue <- uri
+    }
+    return
+
+}
+
+func main() {
+    // rate-limiter
+    rate := time.Second / 500
+    throttle := time.Tick(rate)
+
+    queue := make(chan string)
+
+    go q("itemlist.txt", queue)
+
+    for uri := range queue {
+        <-throttle
+        go fetch(uri, queue)
+    }
+
+}