mild_strip.py

Cleans HTML by removing comments, scripts, styles, and other non-content tags while trimming attribute values to 100 characters for web scraping processing.

from bs4 import BeautifulSoup, Doctype, Tag, Comment


def mild_strip(soup: Tag, keep_d_id: bool = True) -> BeautifulSoup:
    new_soup = BeautifulSoup(str(soup), "html.parser")
    _mild_strip(new_soup, keep_d_id)
    return new_soup


def mild_strip_in_place(soup: BeautifulSoup, keep_d_id: bool = True) -> None:
    _mild_strip(soup, keep_d_id)


def _mild_strip(soup: BeautifulSoup, keep_d_id: bool = True) -> None:
    for element in soup(text=lambda text: isinstance(text, Comment)):
        element.extract()

    # for text in soup.find_all(text=lambda text: isinstance(text, NavigableString)):
    #     if len(text) > 200:
    #         text.replace_with(text[:200] + f"... [{len(text)-200} more chars]")

    for tag in soup(
        ["head", "script", "style", "path", "polygon", "defs", 

... (truncated -- full source via MCP)