lisovskyvlad · February 9, 2014 21:05 · Feb 4, 2014 · Feb 4, 2014 · Jun 6, 2012 · Jun 6, 2012
diff --git a/transcode.go b/transcode.go
@@ -4,64 +4,63 @@
 package main
 
 import (
-	"bufio"
-	"compress/bzip2"
-	"encoding/hex"
-	"flag"
-	"io"
-	"log"
-	"os"
+        "bufio"
+        "encoding/hex"
+        "flag"
+        "io"
+        "log"
+        "os"
 
-	"github.com/dustin/go-humanize"
+        "github.com/dustin/go-humanize"
 )
 
+const hashSize = 20
+
 var saw int64
 
 func maybefatal(err error) {
-	if err != nil {
-		log.Fatalf("Dying with %v", err)
-	}
+        if err != nil {
+                log.Fatalf("Dying with %v", err)
+        }
 }
 
-func process(r *bufio.Reader, w io.Writer) {
-	for {
-		s, err := r.ReadString(byte('\n'))
-		if err == io.EOF {
-			log.Printf("Got an EOF")
-			return
-		}
-		maybefatal(err)
-
-		bytes, err := hex.DecodeString(s[:40])
-		maybefatal(err)
+func process(r io.Reader, w io.Writer) {
+        sc := bufio.NewScanner(r)
+        for sc.Scan() {
+                bytes, err := hex.DecodeString(sc.Text())
+                maybefatal(err)
 
-		n, err := w.Write(bytes)
-		maybefatal(err)
-		if n != 20 {
-			log.Fatalf("Expected to write 20 bytes, wrote %d", n)
-		}
+                n, err := w.Write(bytes)
+                maybefatal(err)
+                if n != 32 {
+                        log.Fatalf("Expected to write %d bytes, wrote %d", hashSize, n)
+                }
 
-		saw += 1
-	}
+                saw += 1
+        }
+        if sc.Err() != nil {
+                log.Fatalf("Error scanning: %v", sc.Err())
+        }
 }
 
 func main() {
-	flag.Parse()
+        flag.Parse()
 
-	infile := flag.Arg(0)
-	outfile := flag.Arg(1)
+        infile := flag.Arg(0)
+        outfile := flag.Arg(1)
 
-	log.Printf("%s -> %s", infile, outfile)
+        log.Printf("%s -> %s", infile, outfile)
 
-	fin, err := os.Open(infile)
-	maybefatal(err)
-	defer fin.Close()
-	b := bufio.NewReader(bzip2.NewReader(fin))
+        fin, err := os.Open(infile)
+        maybefatal(err)
+        defer fin.Close()
 
-	fout, err := os.Create(outfile)
-	maybefatal(err)
-	defer fout.Close()
+        fout, err := os.Create(outfile)
+        maybefatal(err)
+        defer fout.Close()
 
-	process(b, fout)
-	log.Printf("Wrote %s items", humanize.Comma(saw))
-}
+        bw := bufio.NewWriter(fout)
+        defer bw.Flush()
+        process(fin, bw)
+        log.Printf("Wrote %s items", humanize.Comma(saw))
+}
diff --git a/leakedin.go b/leakedin.go
@@ -80,15 +80,15 @@ func searchHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func report() {
-	for {
-		time.Sleep(time.Second)
-
+	for _ = range time.Tick(time.Second) {
 		oldval := int64(0)
 		for !atomic.CompareAndSwapInt64(&processed, oldval, 0) {
 			oldval = processed
 		}
 
-		log.Printf("Processed %v items", oldval)
+		if oldval > 0 {
+			log.Printf("Processed %v items", oldval)
+		}
 	}
 }
 

diff --git a/leakedin.go b/leakedin.go
@@ -1,3 +1,7 @@
+// SHA Hash presence web server.
+//
+// Requires a file containing sorted sha1s in binary form (20 bytes each).
+
 package main
 
 import (

diff --git a/transcode.go b/transcode.go
@@ -0,0 +1,67 @@
+// Given a file containing sorted sha1 hashes in string form, this
+// writes out a new file containing the same hashes in binary form.
+
+package main
+
+import (
+	"bufio"
+	"compress/bzip2"
+	"encoding/hex"
+	"flag"
+	"io"
+	"log"
+	"os"
+
+	"github.com/dustin/go-humanize"
+)
+
+var saw int64
+
+func maybefatal(err error) {
+	if err != nil {
+		log.Fatalf("Dying with %v", err)
+	}
+}
+
+func process(r *bufio.Reader, w io.Writer) {
+	for {
+		s, err := r.ReadString(byte('\n'))
+		if err == io.EOF {
+			log.Printf("Got an EOF")
+			return
+		}
+		maybefatal(err)
+
+		bytes, err := hex.DecodeString(s[:40])
+		maybefatal(err)
+
+		n, err := w.Write(bytes)
+		maybefatal(err)
+		if n != 20 {
+			log.Fatalf("Expected to write 20 bytes, wrote %d", n)
+		}
+
+		saw += 1
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	infile := flag.Arg(0)
+	outfile := flag.Arg(1)
+
+	log.Printf("%s -> %s", infile, outfile)
+
+	fin, err := os.Open(infile)
+	maybefatal(err)
+	defer fin.Close()
+	b := bufio.NewReader(bzip2.NewReader(fin))
+
+	fout, err := os.Create(outfile)
+	maybefatal(err)
+	defer fout.Close()
+
+	process(b, fout)
+	log.Printf("Wrote %s items", humanize.Comma(saw))
+}
diff --git a/leakedin.go b/leakedin.go
@@ -72,7 +72,7 @@ func searchHandler(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(410)
 	}
 
-	fmt.Fprintf(w, `{"leaked": %v}`+"\n", things.Contains(p))
+	fmt.Fprintf(w, `{"leaked": %v}`+"\n", leaked)
 }
 
 func report() {

diff --git a/leakedin.go b/leakedin.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"sort"
+	"sync/atomic"
+	"time"
+)
+
+const item_length = 20
+
+var things thingbytes
+
+var processed int64
+
+type thingbytes []byte
+
+type thinger interface {
+	Get(offset int) string
+	Contains(val string) bool
+	Count() int
+}
+
+func (tb thingbytes) Get(offset int) string {
+	base := offset * item_length
+	return hex.EncodeToString(tb[base : base+item_length])
+}
+
+func (tb thingbytes) Count() int {
+	return len(tb) / item_length
+}
+
+func (tb thingbytes) Contains(needle string) bool {
+	count := tb.Count()
+	off := sort.Search(count, func(x int) bool {
+		return tb.Get(x) > needle
+	})
+
+	return off < count && off > 0 && tb.Get(off-1) == needle
+}
+
+func maybefatal(err error) {
+	if err != nil {
+		log.Fatalf("Dying with %v", err)
+	}
+}
+
+func bytesReader(fn string) thingbytes {
+	b, err := ioutil.ReadFile(fn)
+	maybefatal(err)
+
+	log.Printf("Read all the things (%d bytes)", len(b))
+
+	return thingbytes(b)
+}
+
+func searchHandler(w http.ResponseWriter, r *http.Request) {
+	p := r.URL.Path[1:]
+	atomic.AddInt64(&processed, 1)
+
+	w.Header().Set("Content-type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	leaked := things.Contains(p)
+
+	if !leaked {
+		w.WriteHeader(410)
+	}
+
+	fmt.Fprintf(w, `{"leaked": %v}`+"\n", things.Contains(p))
+}
+
+func report() {
+	for {
+		time.Sleep(time.Second)
+
+		oldval := int64(0)
+		for !atomic.CompareAndSwapInt64(&processed, oldval, 0) {
+			oldval = processed
+		}
+
+		log.Printf("Processed %v items", oldval)
+	}
+}
+
+func main() {
+	things = bytesReader(os.Args[1])
+
+	go report()
+
+	http.HandleFunc("/", searchHandler)
+
+	log.Fatal(http.ListenAndServe(":6262", nil))
+}
No results found