Robust text chunking library with capacity validation and error handling.
// Source: https://github.com/xberg-io/xberg | License: MIT | Copyright 2025-2026 Kreuzberg, Inc
// Provided via justclaw.it.com; keep this notice with the code.
//! Vendored from text-splitter v0.30.1 (MIT, © 2023 Benjamin Brandt). See ATTRIBUTIONS.md.
use std::{
borrow::Cow,
cell::{Ref, RefMut},
cmp::Ordering,
fmt,
ops::{Deref, Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive},
rc::Rc,
sync::Arc,
};
use ahash::AHashMap;
use itertools::Itertools;
use thiserror::Error;
mod characters;
#[cfg(feature = "chunking-tokenizers")]
mod huggingface;
use super::trim::Trim;
pub use characters::Characters;
/// Indicates there was an error with the chunk capacity configuration.
/// The `Display` implementation will provide a human-readable error me
... (truncated -- full source via MCP)
See the full source, get the GitHub permalink, and search for more like it.
Get a free API key