Crawl Window Manager (audit/crawl-window.ts)

Dynamically adjusts fetch concurrency for SEO audits, optimizing politeness and performance.

// Source: https://github.com/every-app/open-seo | License: MIT | Copyright 2026 Ben Senescu
// Provided via justclaw.it.com; keep this notice with the code.
import type { CrawledPageResult } from "@/server/lib/audit/types";

/**
 * Rolling fetch-concurrency window for the crawl. Unlike fixed batches, a
 * slow page only occupies one slot instead of stalling a whole batch. The
 * window adapts to the site: it shrinks when fetches error/block/crawl
 * slowly (politeness toward struggling or defensive sites) and grows when
 * the site answers fast.
 */
interface CrawlWindowLimits {
  /** Window size a chunk starts with, before any observations. */
  initial: number;
  min: number;
  max: number;
  /**
   * Total HTML the in-flight window may buffer at once. Each in-flight page
   * holds its

... (truncated -- full source via MCP)