Skip to content

Instantly share code, notes, and snippets.

@hallelujah-shih

hallelujah-shih/go

Created November 8, 2018 05:53
Show Gist options
  • Save hallelujah-shih/bc12034848c6aadca71de1ccc7f1f625 to your computer and use it in GitHub Desktop.
Save hallelujah-shih/bc12034848c6aadca71de1ccc7f1f625 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. hallelujah-shih created this gist Nov 8, 2018.
    80 changes: 80 additions & 0 deletions go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,80 @@
    package main

    import (
    "compress/gzip"
    "io"
    "log"
    "os"
    "path/filepath"
    "runtime"
    "strings"
    "sync"
    )

    func checkGzip(path string) bool {
    handle, err := os.Open(path)
    if err != nil {
    log.Println(err)
    return false
    }
    defer handle.Close()

    gzReader, err := gzip.NewReader(handle)
    if err != nil {
    log.Println(err)
    return false
    }
    defer gzReader.Close()
    buf := make([]byte, 64*1024)
    for {
    _, err := gzReader.Read(buf)
    if err != nil {
    if err != io.EOF {
    log.Println(err)
    return false
    }
    return true
    }
    }
    return true
    }

    func main() {
    root := "my/gzipfile/dir"
    fileChan := func() chan string {
    fchan := make(chan string)
    go func() {
    defer close(fchan)
    err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
    if err != nil {
    log.Println(err)
    }
    if !info.IsDir() && strings.HasSuffix(path, "gz") {
    fchan <- path
    }
    return nil
    })
    if err != nil {
    log.Fatalln(err)
    }
    }()
    return fchan
    }()

    log.Println("begin")
    var wg sync.WaitGroup
    for i := 0; i < runtime.NumCPU(); i++ {
    wg.Add(1)
    go func(group *sync.WaitGroup, files chan string) {
    defer group.Done()
    for fpath := range files {
    if !checkGzip(fpath) {
    log.Println("remove path:", fpath)
    os.Remove(fpath)
    }
    }
    }(&wg, fileChan)
    }
    wg.Wait()
    log.Println("done")
    }