Tag: weeks

I’m on week 2 of learning go. I just completed the golang.org exercise on creating web crawler (https://tour.golang.org/concurrency/10) and I would please like feedback on my code. It feels overly complicated but I’m a noob so I don’t know.

I came across this post on the same exercies a few minutes ago while writing up this question. The only posted feedback there is one that I already implemented, so I’m hoping to get something different.

I also am not a fan of that implementation because it relies on knowing exactly how many urls will need to be fetched, which works with the fake data but wouldn’t work with real data.

type Fetcher interface {
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls ()string, err error)
}

type FetchState struct {
    sync.Mutex
    fetchedUrls map(string)bool
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher, state *FetchState) {
    if depth <= 0 {
        return
    }

    state.Lock()

    if _, ok := state.fetchedUrls(url); ok {
        state.Unlock()
        return
    }

    state.Unlock()
    body, urls, err := fetcher.Fetch(url)

    if err != nil {
        fmt.Println(err)
        return
    }

    fmt.Printf("found: %s %qn", url, body)
    state.Lock()
    state.fetchedUrls(url) = true
    state.Unlock()

    var wg sync.WaitGroup

    for _, u := range urls {
        wg.Add(1)
        go func(urlToCrawl string, group *sync.WaitGroup) {
            Crawl(urlToCrawl, depth-1, fetcher, state)
            group.Done()
        }(u, &wg)
    }

    wg.Wait()

    return
}

func main() {
    state := &FetchState{
        fetchedUrls: make(map(string)bool),
    }

    Crawl("https://golang.org/", 4, fetcher, state)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map(string)*fakeResult

type fakeResult struct {
    body string
    urls ()string
}

func (f fakeFetcher) Fetch(url string) (string, ()string, error) {
    time.Sleep(1000 * time.Millisecond)

    if res, ok := f(url); ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
    "https://golang.org/": &fakeResult{
        "The Go Programming Language",
        ()string{
            "https://golang.org/pkg/",
            "https://golang.org/cmd/",
        },
    },
    "https://golang.org/pkg/": &fakeResult{
        "Packages",
        ()string{
            "https://golang.org/",
            "https://golang.org/cmd/",
            "https://golang.org/pkg/fmt/",
            "https://golang.org/pkg/os/",
        },
    },
    "https://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        ()string{
            "https://golang.org/",
            "https://golang.org/pkg/",
        },
    },
    "https://golang.org/cmd/": &fakeResult{
        "Package cmd",
        ()string{
        },
    },
    "https://golang.org/pkg/os/": &fakeResult{
        "Package os",
        ()string{
            "https://golang.org/",
            "https://golang.org/pkg/",
        },
    },
}