LanceDB IPC Stream Utility (remote/util.rs)

Utilities for streaming Arrow record batches as compressed IPC data. Use this for efficient data transfer between LanceDB clients and servers.

// Source: https://github.com/lancedb/lancedb | License: Apache-2.0 | Copyright You must retain, in the Source form of any Derivative Works
// Provided via justclaw.it.com; keep this notice with the code.
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors

use arrow_ipc::CompressionType;
use futures::{Stream, StreamExt};
use reqwest::Response;

use crate::{Result, arrow::SendableRecordBatchStream};

use super::db::ServerVersion;

pub fn stream_as_ipc(
    data: SendableRecordBatchStream,
) -> Result<impl Stream<Item = Result<bytes::Bytes>>> {
    let options = arrow_ipc::writer::IpcWriteOptions::default()
        .try_with_compression(Some(CompressionType::LZ4_FRAME))?;
    const WRITE_BUF_SIZE: usize = 4096;
    let buf = Vec::with_capacity(WRI

... (truncated -- full source via MCP)