Scrapy AutoThrottle Extension (extensions/throttle.py)

Automatically adjusts Scrapy crawler download speeds based on server load and latency. Reuse to prevent target site overload and respect crawl limits.

# Source: https://github.com/scrapy/scrapy | License: BSD-3-Clause | Copyright Scrapy developers
# Provided via justclaw.it.com; keep this notice with the code.
from __future__ import annotations

import logging
from typing import TYPE_CHECKING, Self

from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.response import _body_size

if TYPE_CHECKING:
    from scrapy.core.downloader import Slot
    from scrapy.crawler import Crawler
    from scrapy.http import Response


logger = logging.getLogger(__name__)


class AutoThrottle:
    def __init__(self, crawler: Crawler):
        self.crawler: Crawler = crawler
        if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"):
            raise NotConfigured

        self.debug: bool = crawler.

... (truncated -- full source via MCP)