chuchao333 · April 11, 2020 18:33 · Apr 11, 2020
diff --git a/exercise_web_crawler.go b/exercise_web_crawler.go
@@ -0,0 +1,95 @@
+package main
+
+import (
+	"fmt"
+	"sync"
+)
+
+type Fetcher interface {
+	// Fetch returns the body of URL and
+	// a slice of URLs found on that page.
+	Fetch(url string) (body string, urls []string, err error)
+}
+
+// Crawl uses fetcher to recursively crawl
+// pages starting with url, to a maximum of depth.
+func Crawl(url string, depth int, fetcher Fetcher, fetched *sync.Map, crawlers *sync.WaitGroup) {
+	defer crawlers.Done()
+
+	if depth <= 0 {
+		return
+	}
+	_, alreadyFetched := fetched.Load(url)
+	if alreadyFetched {
+	    return
+	}
+	body, urls, err := fetcher.Fetch(url)
+	fetched.Store(url, true)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Printf("found: %s %q\n", url, body)
+	for _, u := range urls {
+	    crawlers.Add(1)
+		go Crawl(u, depth-1, fetcher, fetched, crawlers)
+	}
+	return
+}
+
+func main() {
+    alreadyFetched := new (sync.Map)
+	crawlers := new (sync.WaitGroup)
+	crawlers.Add(1)
+	go Crawl("https://golang.org/", 4, fetcher, alreadyFetched, crawlers)
+	crawlers.Wait()
+}
+
+// fakeFetcher is Fetcher that returns canned results.
+type fakeFetcher map[string]*fakeResult
+
+type fakeResult struct {
+	body string
+	urls []string
+}
+
+func (f fakeFetcher) Fetch(url string) (string, []string, error) {
+	if res, ok := f[url]; ok {
+		return res.body, res.urls, nil
+	}
+	return "", nil, fmt.Errorf("not found: %s", url)
+}
+
+// fetcher is a populated fakeFetcher.
+var fetcher = fakeFetcher{
+	"https://golang.org/": &fakeResult{
+		"The Go Programming Language",
+		[]string{
+			"https://golang.org/pkg/",
+			"https://golang.org/cmd/",
+		},
+	},
+	"https://golang.org/pkg/": &fakeResult{
+		"Packages",
+		[]string{
+			"https://golang.org/",
+			"https://golang.org/cmd/",
+			"https://golang.org/pkg/fmt/",
+			"https://golang.org/pkg/os/",
+		},
+	},
+	"https://golang.org/pkg/fmt/": &fakeResult{
+		"Package fmt",
+		[]string{
+			"https://golang.org/",
+			"https://golang.org/pkg/",
+		},
+	},
+	"https://golang.org/pkg/os/": &fakeResult{
+		"Package os",
+		[]string{
+			"https://golang.org/",
+			"https://golang.org/pkg/",
+		},
+	},
+}