Parallel Shared Memory Buffer (content_iterator/buffer.py)

Provides a high-performance buffer backed by shared memory for concurrent object downloads. Use this to manage large data transfers across multiple worker processes efficiently.

# Source: https://github.com/NVIDIA/aistore | License: MIT | Copyright 2017 NVIDIA Corporation
# Provided via justclaw.it.com; keep this notice with the code.
#
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#

import multiprocessing as mp
import os
from multiprocessing import shared_memory
from typing import List


class SharedMemory(shared_memory.SharedMemory):
    """Shared memory whose finalizer closes the descriptor despite live views."""

    # TODO: remove when supported Python versions include the fix for
    # https://github.com/python/cpython/issues/155003 (PR #155070).
    def __del__(self):
        try:
            if os.name == "posix" and self._fd >= 0:
                os.close(self._fd)
                self._fd = -1
        except OSError:
            pass




... (truncated -- full source via MCP)