-
-
Save lisovskyvlad/8905912 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // SHA Hash presence web server. | |
| // | |
| // Requires a file containing sorted sha1s in binary form (20 bytes each). | |
| package main | |
| import ( | |
| "encoding/hex" | |
| "fmt" | |
| "io/ioutil" | |
| "log" | |
| "net/http" | |
| "os" | |
| "sort" | |
| "sync/atomic" | |
| "time" | |
| ) | |
| const item_length = 20 | |
| var things thingbytes | |
| var processed int64 | |
| type thingbytes []byte | |
| type thinger interface { | |
| Get(offset int) string | |
| Contains(val string) bool | |
| Count() int | |
| } | |
| func (tb thingbytes) Get(offset int) string { | |
| base := offset * item_length | |
| return hex.EncodeToString(tb[base : base+item_length]) | |
| } | |
| func (tb thingbytes) Count() int { | |
| return len(tb) / item_length | |
| } | |
| func (tb thingbytes) Contains(needle string) bool { | |
| count := tb.Count() | |
| off := sort.Search(count, func(x int) bool { | |
| return tb.Get(x) > needle | |
| }) | |
| return off < count && off > 0 && tb.Get(off-1) == needle | |
| } | |
| func maybefatal(err error) { | |
| if err != nil { | |
| log.Fatalf("Dying with %v", err) | |
| } | |
| } | |
| func bytesReader(fn string) thingbytes { | |
| b, err := ioutil.ReadFile(fn) | |
| maybefatal(err) | |
| log.Printf("Read all the things (%d bytes)", len(b)) | |
| return thingbytes(b) | |
| } | |
| func searchHandler(w http.ResponseWriter, r *http.Request) { | |
| p := r.URL.Path[1:] | |
| atomic.AddInt64(&processed, 1) | |
| w.Header().Set("Content-type", "application/json") | |
| w.Header().Set("Access-Control-Allow-Origin", "*") | |
| leaked := things.Contains(p) | |
| if !leaked { | |
| w.WriteHeader(410) | |
| } | |
| fmt.Fprintf(w, `{"leaked": %v}`+"\n", leaked) | |
| } | |
| func report() { | |
| for _ = range time.Tick(time.Second) { | |
| oldval := int64(0) | |
| for !atomic.CompareAndSwapInt64(&processed, oldval, 0) { | |
| oldval = processed | |
| } | |
| if oldval > 0 { | |
| log.Printf("Processed %v items", oldval) | |
| } | |
| } | |
| } | |
| func main() { | |
| things = bytesReader(os.Args[1]) | |
| go report() | |
| http.HandleFunc("/", searchHandler) | |
| log.Fatal(http.ListenAndServe(":6262", nil)) | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Given a file containing sorted sha1 hashes in string form, this | |
| // writes out a new file containing the same hashes in binary form. | |
| package main | |
| import ( | |
| "bufio" | |
| "compress/bzip2" | |
| "encoding/hex" | |
| "flag" | |
| "io" | |
| "log" | |
| "os" | |
| "github.com/dustin/go-humanize" | |
| ) | |
| var saw int64 | |
| func maybefatal(err error) { | |
| if err != nil { | |
| log.Fatalf("Dying with %v", err) | |
| } | |
| } | |
| func process(r *bufio.Reader, w io.Writer) { | |
| for { | |
| s, err := r.ReadString(byte('\n')) | |
| if err == io.EOF { | |
| log.Printf("Got an EOF") | |
| return | |
| } | |
| maybefatal(err) | |
| bytes, err := hex.DecodeString(s[:40]) | |
| maybefatal(err) | |
| n, err := w.Write(bytes) | |
| maybefatal(err) | |
| if n != 20 { | |
| log.Fatalf("Expected to write 20 bytes, wrote %d", n) | |
| } | |
| saw += 1 | |
| } | |
| } | |
| func main() { | |
| flag.Parse() | |
| infile := flag.Arg(0) | |
| outfile := flag.Arg(1) | |
| log.Printf("%s -> %s", infile, outfile) | |
| fin, err := os.Open(infile) | |
| maybefatal(err) | |
| defer fin.Close() | |
| b := bufio.NewReader(bzip2.NewReader(fin)) | |
| fout, err := os.Create(outfile) | |
| maybefatal(err) | |
| defer fout.Close() | |
| process(b, fout) | |
| log.Printf("Wrote %s items", humanize.Comma(saw)) | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment