Hey, regarding to the latest Version of the Crawler, it looks like it wont finish.
import (
"fmt"
"log"
"github.com/rs/zerolog"
"github.com/twiny/wbot"
"github.com/twiny/wbot/pkg/api"
)
func main() {
bot := wbot.New(
wbot.WithParallel(50),
wbot.WithMaxDepth(5),
wbot.WithRateLimit(&api.RateLimit{
Hostname: "*",
Rate: "10/1s",
}),
wbot.WithLogLevel(zerolog.DebugLevel),
)
defer bot.Shutdown()
// read responses
bot.OnReponse(func(resp *api.Response) {
fmt.Printf("crawled: %s\n", resp.URL.String())
})
if err := bot.Run(
"https://example.com",
); err != nil {
log.Fatal(err)
}
log.Printf("finished crawling\n")
}
2024-03-24T11:05:28-04:00 DBG fetched target=https://example.com
crawled: https://example.com
2024-03-24T11:07:00-04:00 INF Starting crawler with 1 links
2024-03-24T11:07:10-04:00 ERR fetch error="context deadline exceeded" target=https://examdfasdfasdfple.com