Request Queue Client (_base/_request_queue_client.py)

crawlee-python · src/crawlee/storage_clients/_base/_request_queue_client.py · python · Apache-2.0
scraping, database, lang:python, tag:crawlee, tag:abstract-class

Abstract base class for managing request queue resources within a storage client. Reuse when implementing custom storage backends for web crawling request queues.

# Source: https://github.com/apify/crawlee-python | License: Apache-2.0 | Copyright You must retain, in the Source form of any Derivative Works
# Provided via justclaw.it.com; keep this notice with the code.
from __future__ import annotations

from abc import ABC, abstractmethod
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from collections.abc import Sequence

    from crawlee import Request
    from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata


class RequestQueueClient(ABC):
    """An abstract class for request queue resource clients.

    These clients are specific to the type of resource they manage and operate under a designated storage
    client, like a memory storage client.
    """

    @abstractmethod
    async def g

... (truncated -- full source via MCP)

See the full source, get the GitHub permalink, and search for more like it.

Get a free API key