Google Git
Sign in
apache / arrow-site / 1e645257273c2521a371004aa4d87af19a1c47bc / . / docs / dev / _modules / pyarrow / ipc.html
blob: e1385dd864b09a5dad733ca7f23eabc2d9fc34f5 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" data-content_root="../../" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>pyarrow.ipc &#8212; Apache Arrow v23.0.0.dev37</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=03e43079" />
<link rel="stylesheet" type="text/css" href="../../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../../_static/sphinx-design.min.css?v=95c83b7e" />
<link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=8dcd28dc" />
<!-- So that users can add custom icons -->
<script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../../_static/documentation_options.js?v=9fc6757a"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../../_static/copybutton.js?v=3bb21c8c"></script>
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
<script type="module" src="https://cdn.jsdelivr.net/npm/mermaid@11.2.0/dist/mermaid.esm.min.mjs"></script>
<script type="module" src="https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk@0.1.4/dist/mermaid-layout-elk.esm.min.mjs"></script>
<script type="module">import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@11.2.0/dist/mermaid.esm.min.mjs";import elkLayouts from "https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk@0.1.4/dist/mermaid-layout-elk.esm.min.mjs";mermaid.registerLayoutLoaders(elkLayouts);mermaid.initialize({startOnLoad:false});</script>
<script type="module">
import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@11.2.0/dist/mermaid.esm.min.mjs";
window.addEventListener("load", () => mermaid.run());
</script>
<script>DOCUMENTATION_OPTIONS.pagename = '_modules/pyarrow/ipc';</script>
<script>
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json';
DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/';
DOCUMENTATION_OPTIONS.show_version_warning_banner =
true;
</script>
<link rel="canonical" href="https://arrow.apache.org/docs/_modules/pyarrow/ipc.html" />
<link rel="icon" href="../../_static/favicon.ico"/>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="23.0.0.dev37" />
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
/* We explicitly disable cookie tracking to avoid privacy issues */
_paq.push(['disableCookies']);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '20']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class=" navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../../index.html">
<img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v23.0.0.dev37 - Home"/>
<img src="../../_static/arrow-dark.png" class="logo__image only-dark pst-js-only" alt="Apache Arrow v23.0.0.dev37 - Home"/>
</a></div>
</div>
<div class=" navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../developers/index.html">
Development
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../implementations.html">
Implementations
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item"><div class="kapa-ai-bot">
<script
async
src="https://widget.kapa.ai/kapa-widget.bundle.js"
data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2"
data-project-name="Apache Arrow"
data-project-color="#000000"
data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png"
data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc."
data-consent-required="true"
data-user-analytics-cookie-enabled="false"
data-consent-screen-disclaimer="By clicking &quot;I agree, let's chat&quot;, you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies."
></script>
</div>
</div>
<div class="navbar-item">
<div class="version-switcher__container dropdown pst-js-only">
<button id="pst-version-switcher-button-2"
type="button"
class="version-switcher__button btn btn-sm dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-2"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-2"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-2">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i>
<span class="sr-only">LinkedIn</span></a>
</li>
<li class="nav-item">
<a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i>
<span class="sr-only">BlueSky</span></a>
</li>
</ul></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar hide-on-wide">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../developers/index.html">
Development
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../implementations.html">
Implementations
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item"><div class="kapa-ai-bot">
<script
async
src="https://widget.kapa.ai/kapa-widget.bundle.js"
data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2"
data-project-name="Apache Arrow"
data-project-color="#000000"
data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png"
data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc."
data-consent-required="true"
data-user-analytics-cookie-enabled="false"
data-consent-screen-disclaimer="By clicking &quot;I agree, let's chat&quot;, you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies."
></script>
</div>
</div>
<div class="navbar-item">
<div class="version-switcher__container dropdown pst-js-only">
<button id="pst-version-switcher-button-3"
type="button"
class="version-switcher__button btn btn-sm dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-3"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-3"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-3">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i>
<span class="sr-only">LinkedIn</span></a>
</li>
<li class="nav-item">
<a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i>
<span class="sr-only">BlueSky</span></a>
</li>
</ul></div>
</div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../index.html" class="nav-link">Module code</a></li>
<li class="breadcrumb-item"><a href="../pyarrow.html" class="nav-link">pyarrow</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">pyarrow.ipc</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<h1>Source code for pyarrow.ipc</h1><div class="highlight"><pre>
<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
<span class="c1"># distributed with this work for additional information</span>
<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
<span class="c1"># Arrow file and stream reader/writer classes, and other messaging tools</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">IpcReadOptions</span><span class="p">,</span> <span class="n">IpcWriteOptions</span><span class="p">,</span> <span class="n">ReadStats</span><span class="p">,</span> <span class="n">WriteStats</span><span class="p">,</span> <span class="c1"># noqa</span>
<span class="n">Message</span><span class="p">,</span> <span class="n">MessageReader</span><span class="p">,</span>
<span class="n">RecordBatchReader</span><span class="p">,</span> <span class="n">_ReadPandasMixin</span><span class="p">,</span>
<span class="n">MetadataVersion</span><span class="p">,</span> <span class="n">Alignment</span><span class="p">,</span>
<span class="n">read_message</span><span class="p">,</span> <span class="n">read_record_batch</span><span class="p">,</span> <span class="n">read_schema</span><span class="p">,</span>
<span class="n">read_tensor</span><span class="p">,</span> <span class="n">write_tensor</span><span class="p">,</span>
<span class="n">get_record_batch_size</span><span class="p">,</span> <span class="n">get_tensor_size</span><span class="p">)</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">lib</span>
<div class="viewcode-block" id="RecordBatchStreamReader">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamReader.html#pyarrow.ipc.RecordBatchStreamReader">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">RecordBatchStreamReader</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchStreamReader</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Reader for the Arrow streaming binary format.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span>
<span class="sd"> Either an in-memory buffer, or a readable file object.</span>
<span class="sd"> If you want to use memory map use MemoryMappedFile as source.</span>
<span class="sd"> options : pyarrow.ipc.IpcReadOptions</span>
<span class="sd"> Options for IPC deserialization.</span>
<span class="sd"> If None, default values will be used.</span>
<span class="sd"> memory_pool : MemoryPool, default None</span>
<span class="sd"> If None, default memory pool is used.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="RecordBatchStreamReader.__init__">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamReader.html#pyarrow.ipc.RecordBatchStreamReader.__init__">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">options</span> <span class="o">=</span> <span class="n">_ensure_default_ipc_read_options</span><span class="p">(</span><span class="n">options</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div>
</div>
<span class="n">_ipc_writer_class_doc</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span><span class="se">\</span>
<span class="s2">Parameters</span>
<span class="s2">----------</span>
<span class="s2">sink : str, pyarrow.NativeFile, or file-like Python object</span>
<span class="s2"> Either a file path, or a writable file object.</span>
<span class="s2">schema : pyarrow.Schema</span>
<span class="s2"> The Arrow schema for data to be written to the file.</span>
<span class="s2">options : pyarrow.ipc.IpcWriteOptions</span>
<span class="s2"> Options for IPC serialization.</span>
<span class="s2"> If None, default values will be used: the legacy format will not</span>
<span class="s2"> be used unless overridden by setting the environment variable</span>
<span class="s2"> ARROW_PRE_0_15_IPC_FORMAT=1, and the V5 metadata version will be</span>
<span class="s2"> used unless overridden by setting the environment variable</span>
<span class="s2"> ARROW_PRE_1_0_METADATA_VERSION=1.&quot;&quot;&quot;</span>
<span class="n">_ipc_file_writer_class_doc</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">_ipc_writer_class_doc</span>
<span class="o">+</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="s2">&quot;&quot;&quot;</span><span class="se">\</span>
<span class="s2">metadata : dict | pyarrow.KeyValueMetadata, optional</span>
<span class="s2"> Key/value pairs (both must be bytes-like) that will be stored</span>
<span class="s2"> in the file footer and are retrievable via</span>
<span class="s2"> pyarrow.ipc.open_file(...).metadata.&quot;&quot;&quot;</span>
<span class="p">)</span>
<div class="viewcode-block" id="RecordBatchStreamWriter">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamWriter.html#pyarrow.ipc.RecordBatchStreamWriter">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchStreamWriter</span><span class="p">):</span>
<span class="vm">__doc__</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;&quot;&quot;Writer for the Arrow streaming binary format</span>
<span class="si">{</span><span class="n">_ipc_writer_class_doc</span><span class="si">}</span><span class="s2">&quot;&quot;&quot;</span>
<div class="viewcode-block" id="RecordBatchStreamWriter.__init__">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamWriter.html#pyarrow.ipc.RecordBatchStreamWriter.__init__">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">options</span> <span class="o">=</span> <span class="n">_get_legacy_format_default</span><span class="p">(</span><span class="n">options</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span></div>
</div>
<div class="viewcode-block" id="RecordBatchFileReader">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileReader.html#pyarrow.ipc.RecordBatchFileReader">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">RecordBatchFileReader</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchFileReader</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Class for reading Arrow record batch data from the Arrow binary file format</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span>
<span class="sd"> Either an in-memory buffer, or a readable file object.</span>
<span class="sd"> If you want to use memory map use MemoryMappedFile as source.</span>
<span class="sd"> footer_offset : int, default None</span>
<span class="sd"> If the file is embedded in some larger file, this is the byte offset to</span>
<span class="sd"> the very end of the file data</span>
<span class="sd"> options : pyarrow.ipc.IpcReadOptions</span>
<span class="sd"> Options for IPC serialization.</span>
<span class="sd"> If None, default values will be used.</span>
<span class="sd"> memory_pool : MemoryPool, default None</span>
<span class="sd"> If None, default memory pool is used.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="RecordBatchFileReader.__init__">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileReader.html#pyarrow.ipc.RecordBatchFileReader.__init__">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">options</span> <span class="o">=</span> <span class="n">_ensure_default_ipc_read_options</span><span class="p">(</span><span class="n">options</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="n">footer_offset</span><span class="p">,</span>
<span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div>
</div>
<div class="viewcode-block" id="RecordBatchFileWriter">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileWriter.html#pyarrow.ipc.RecordBatchFileWriter">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">RecordBatchFileWriter</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchFileWriter</span><span class="p">):</span>
<span class="vm">__doc__</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;&quot;&quot;Writer to create the Arrow binary file format</span>
<span class="si">{</span><span class="n">_ipc_file_writer_class_doc</span><span class="si">}</span><span class="s2">&quot;&quot;&quot;</span>
<div class="viewcode-block" id="RecordBatchFileWriter.__init__">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileWriter.html#pyarrow.ipc.RecordBatchFileWriter.__init__">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">options</span> <span class="o">=</span> <span class="n">_get_legacy_format_default</span><span class="p">(</span><span class="n">options</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="n">metadata</span><span class="p">)</span></div>
</div>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_legacy_format_default</span><span class="p">(</span><span class="n">options</span><span class="p">):</span>
<span class="k">if</span> <span class="n">options</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">options</span><span class="p">,</span> <span class="n">IpcWriteOptions</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;expected IpcWriteOptions, got </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">options</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">options</span>
<span class="n">metadata_version</span> <span class="o">=</span> <span class="n">MetadataVersion</span><span class="o">.</span><span class="n">V5</span>
<span class="n">use_legacy_format</span> <span class="o">=</span> \
<span class="nb">bool</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;ARROW_PRE_0_15_IPC_FORMAT&#39;</span><span class="p">,</span> <span class="s1">&#39;0&#39;</span><span class="p">)))</span>
<span class="k">if</span> <span class="nb">bool</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;ARROW_PRE_1_0_METADATA_VERSION&#39;</span><span class="p">,</span> <span class="s1">&#39;0&#39;</span><span class="p">))):</span>
<span class="n">metadata_version</span> <span class="o">=</span> <span class="n">MetadataVersion</span><span class="o">.</span><span class="n">V4</span>
<span class="k">return</span> <span class="n">IpcWriteOptions</span><span class="p">(</span><span class="n">use_legacy_format</span><span class="o">=</span><span class="n">use_legacy_format</span><span class="p">,</span>
<span class="n">metadata_version</span><span class="o">=</span><span class="n">metadata_version</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_ensure_default_ipc_read_options</span><span class="p">(</span><span class="n">options</span><span class="p">):</span>
<span class="k">if</span> <span class="n">options</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">options</span><span class="p">,</span> <span class="n">IpcReadOptions</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;expected IpcReadOptions, got </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">options</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">options</span> <span class="ow">or</span> <span class="n">IpcReadOptions</span><span class="p">()</span>
<div class="viewcode-block" id="new_stream">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.new_stream.html#pyarrow.ipc.new_stream">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">new_stream</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="n">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span>
<span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span></div>
<span class="n">new_stream</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;&quot;&quot;</span><span class="se">\</span>
<span class="s2">Create an Arrow columnar IPC stream writer instance</span>
<span class="si">{</span><span class="n">_ipc_writer_class_doc</span><span class="si">}</span>
<span class="s2">Returns</span>
<span class="s2">-------</span>
<span class="s2">writer : RecordBatchStreamWriter</span>
<span class="s2"> A writer for the given sink</span>
<span class="s2">&quot;&quot;&quot;</span>
<div class="viewcode-block" id="open_stream">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.open_stream.html#pyarrow.ipc.open_stream">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">open_stream</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Create reader for Arrow streaming format.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span>
<span class="sd"> Either an in-memory buffer, or a readable file object.</span>
<span class="sd"> options : pyarrow.ipc.IpcReadOptions</span>
<span class="sd"> Options for IPC serialization.</span>
<span class="sd"> If None, default values will be used.</span>
<span class="sd"> memory_pool : MemoryPool, default None</span>
<span class="sd"> If None, default memory pool is used.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> reader : RecordBatchStreamReader</span>
<span class="sd"> A reader for the given source</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">RecordBatchStreamReader</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span>
<span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div>
<div class="viewcode-block" id="new_file">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.new_file.html#pyarrow.ipc.new_file">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">new_file</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="n">RecordBatchFileWriter</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="n">metadata</span><span class="p">)</span></div>
<span class="n">new_file</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;&quot;&quot;</span><span class="se">\</span>
<span class="s2">Create an Arrow columnar IPC file writer instance</span>
<span class="si">{</span><span class="n">_ipc_file_writer_class_doc</span><span class="si">}</span>
<span class="s2">Returns</span>
<span class="s2">-------</span>
<span class="s2">writer : RecordBatchFileWriter</span>
<span class="s2"> A writer for the given sink</span>
<span class="s2">&quot;&quot;&quot;</span>
<div class="viewcode-block" id="open_file">
<a class="viewcode-back" href="../../python/generated/pyarrow.ipc.open_file.html#pyarrow.ipc.open_file">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">open_file</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Create reader for Arrow file format.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span>
<span class="sd"> Either an in-memory buffer, or a readable file object.</span>
<span class="sd"> footer_offset : int, default None</span>
<span class="sd"> If the file is embedded in some larger file, this is the byte offset to</span>
<span class="sd"> the very end of the file data.</span>
<span class="sd"> options : pyarrow.ipc.IpcReadOptions</span>
<span class="sd"> Options for IPC serialization.</span>
<span class="sd"> If None, default values will be used.</span>
<span class="sd"> memory_pool : MemoryPool, default None</span>
<span class="sd"> If None, default memory pool is used.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> reader : RecordBatchFileReader</span>
<span class="sd"> A reader for the given source</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">RecordBatchFileReader</span><span class="p">(</span>
<span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="n">footer_offset</span><span class="p">,</span>
<span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">serialize_pandas</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">nthreads</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">preserve_index</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Serialize a pandas DataFrame into a buffer protocol compatible object.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> df : pandas.DataFrame</span>
<span class="sd"> nthreads : int, default None</span>
<span class="sd"> Number of threads to use for conversion to Arrow, default all CPUs.</span>
<span class="sd"> preserve_index : bool, default None</span>
<span class="sd"> The default of None will store the index as a column, except for</span>
<span class="sd"> RangeIndex which is stored as metadata only. If True, always</span>
<span class="sd"> preserve the pandas index data as a column. If False, no index</span>
<span class="sd"> information is saved and the result will have a default RangeIndex.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> buf : buffer</span>
<span class="sd"> An object compatible with the buffer protocol.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_pandas</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
<span class="n">preserve_index</span><span class="o">=</span><span class="n">preserve_index</span><span class="p">)</span>
<span class="n">sink</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">BufferOutputStream</span><span class="p">()</span>
<span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">batch</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span>
<span class="n">writer</span><span class="o">.</span><span class="n">write_batch</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>
<span class="k">return</span> <span class="n">sink</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">deserialize_pandas</span><span class="p">(</span><span class="n">buf</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Deserialize a buffer protocol compatible object into a pandas DataFrame.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> buf : buffer</span>
<span class="sd"> An object compatible with the buffer protocol.</span>
<span class="sd"> use_threads : bool, default True</span>
<span class="sd"> Whether to parallelize the conversion using multiple threads.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> df : pandas.DataFrame</span>
<span class="sd"> The buffer deserialized as pandas DataFrame</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">buffer_reader</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">BufferReader</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>
<span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchStreamReader</span><span class="p">(</span><span class="n">buffer_reader</span><span class="p">)</span> <span class="k">as</span> <span class="n">reader</span><span class="p">:</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">reader</span><span class="o">.</span><span class="n">read_all</span><span class="p">()</span>
<span class="k">return</span> <span class="n">table</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span><span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span>
</pre></div>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
</div>
</footer>
</div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 2016-2025 Apache Software Foundation.
Apache Arrow, Arrow, Apache, the Apache logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
<!-- # L10n: Setting the PST URL as an argument as this does not need to be localized -->
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.16.1.
</p></div>
</div>
</div>
</footer>
</body>
</html>