kylelemons · October 7, 2011 01:55 · Oct 7, 2011
diff --git a/gotour-69.go b/gotour-69.go
@@ -0,0 +1,106 @@
+package main
+
+import (
+	"os"
+	"fmt"
+)
+
+type Fetcher interface {
+	// Fetch returns the body of URL and
+	// a slice of URLs found on that page.
+	Fetch(url string) (body string, urls []string, err os.Error)
+}
+
+// Crawl uses fetcher to recursively crawl
+// pages starting with url, to a maximum of depth.
+func Crawl(url string, depth int, fetcher Fetcher) {
+	type moreUrls struct{
+		depth int
+		urls  []string
+	}
+	more := make(chan moreUrls)
+
+	getPage := func(url string, depth int) {
+		body, urls, err := fetcher.Fetch(url)
+		if err != nil {
+			fmt.Println(err)
+		} else {
+			fmt.Printf("found[%d:%s] %q\n", depth, url, body)
+		}
+		more <- moreUrls{depth+1, urls}
+	}
+
+	outstanding := 1
+	go getPage(url, 0)
+
+	visited := map[string]bool{url:true}
+	for outstanding > 0 {
+		next := <-more
+		outstanding--
+		if next.depth > depth {
+			continue
+		}
+		for _, url := range next.urls {
+			if _, seen := visited[url]; seen {
+				continue
+			}
+			visited[url] = true
+			outstanding++
+			go getPage(url, next.depth)
+		}
+	}
+}
+
+func main() {
+	Crawl("http://golang.org/", 4, fetcher)
+}
+
+
+// fakeFetcher is Fetcher that returns canned results.
+type fakeFetcher map[string]*fakeResult
+
+type fakeResult struct {
+	body string
+	urls     []string
+}
+
+func (f *fakeFetcher) Fetch(url string) (string, []string, os.Error) {
+	if res, ok := (*f)[url]; ok {
+		return res.body, res.urls, nil
+	}
+	return "", nil, fmt.Errorf("not found: %s", url)
+}
+
+// fetcher is a populated fakeFetcher.
+var fetcher = &fakeFetcher{
+	"http://golang.org/": &fakeResult{
+		"The Go Programming Language",
+		[]string{
+			"http://golang.org/pkg/",
+			"http://golang.org/cmd/",
+		},
+	},
+	"http://golang.org/pkg/": &fakeResult{
+		"Packages",
+		[]string{
+			"http://golang.org/",
+			"http://golang.org/cmd/",
+			"http://golang.org/pkg/fmt/",
+			"http://golang.org/pkg/os/",
+		},
+	},
+	"http://golang.org/pkg/fmt/": &fakeResult{
+		"Package fmt",
+		[]string{
+			"http://golang.org/",
+			"http://golang.org/pkg/",
+		},
+	},
+	"http://golang.org/pkg/os/": &fakeResult{
+		"Package os",
+		[]string{
+			"http://golang.org/",
+			"http://golang.org/pkg/",
+		},
+	},
+}
diff --git a/xoutput.txt b/xoutput.txt
@@ -0,0 +1,5 @@
+found[0:http://golang.org/] "The Go Programming Language"
+found[1:http://golang.org/pkg/] "Packages"
+not found: http://golang.org/cmd/
+found[2:http://golang.org/pkg/fmt/] "Package fmt"
+found[2:http://golang.org/pkg/os/] "Package os"