WithMaxDepth (scraper/options.go)

Provides functional options to configure a web scraper's behavior, such as depth, concurrency, and URL blacklisting.

// Source: https://github.com/tmc/langchaingo | License: MIT | Copyright Travis Cline <travis.cline@gmail.com>
// Provided via justclaw.it.com; keep this notice with the code.
package scraper

type Options func(*Scraper)

// WithMaxDepth sets the maximum depth for the Scraper.
//
// Default value: 1
//
// maxDepth: the maximum depth to set.
// Returns: an Options function.
func WithMaxDepth(maxDepth int) Options {
	return func(o *Scraper) {
		o.MaxDepth = maxDepth
	}
}

// WithParallelsNum sets the number of maximum allowed concurrent
// requests of the matching domains
//
// Default value: 2
//
// parallels: the number of parallels to set.
// Returns: the updated Scraper options.
func WithParallelsNum(parallels int) Options {
	return func(o *Scraper) {
		o.Parallels = parallels
	}
}

// W

... (truncated -- full source via MCP)