blob: 19db436cc9d2bbc4e4cfd84f3c39c12ee9c008d6 [file] [log] [blame]
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Serialization and IPC &mdash; Apache Arrow v2.0.0</title>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<!--[if lt IE 9]>
<script src="../../_static/js/html5shiv.min.js"></script>
<![endif]-->
<script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
<script src="../../_static/jquery.js"></script>
<script src="../../_static/underscore.js"></script>
<script src="../../_static/doctools.js"></script>
<script src="../../_static/language_data.js"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<link rel="canonical" href="https://arrow.apache.org/docs/python/api/ipc.html" />
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="pyarrow.ipc.new_file" href="../generated/pyarrow.ipc.new_file.html" />
<link rel="prev" title="pyarrow.Tensor" href="../generated/pyarrow.Tensor.html" />
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["setDoNotTrack", true]);
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '20']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../index.html" class="icon icon-home" alt="Documentation Home"> Apache Arrow
</a>
<div class="version">
2.0.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<p class="caption"><span class="caption-text">Specifications and Protocols</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../format/Versioning.html">Format Versioning and Stability</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../format/Columnar.html">Arrow Columnar Format</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../format/Flight.html">Arrow Flight RPC</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../format/Integration.html">Integration Testing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../format/CDataInterface.html">The Arrow C data interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../format/CStreamInterface.html">The Arrow C stream interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../format/Other.html">Other Data Structures</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../status.html">Implementation Status</a></li>
<li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/docs/c_glib/">C/GLib</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cpp/index.html">C++</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/master/csharp/README.md">C#</a></li>
<li class="toctree-l1"><a class="reference external" href="https://godoc.org/github.com/apache/arrow/go/arrow">Go</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../java/index.html">Java</a></li>
<li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/docs/js/">JavaScript</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/master/matlab/README.md">MATLAB</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../index.html">Python</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../install.html">Installing PyArrow</a></li>
<li class="toctree-l2"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data.html">Data Types and In-Memory Data Model</a></li>
<li class="toctree-l2"><a class="reference internal" href="../compute.html">Compute Functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ipc.html">Streaming, Serialization, and IPC</a></li>
<li class="toctree-l2"><a class="reference internal" href="../filesystems.html">Filesystem Interface</a></li>
<li class="toctree-l2"><a class="reference internal" href="../filesystems_deprecated.html">Filesystem Interface (legacy)</a></li>
<li class="toctree-l2"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li>
<li class="toctree-l2"><a class="reference internal" href="../numpy.html">NumPy Integration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../pandas.html">Pandas Integration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../timestamps.html">Timestamps</a></li>
<li class="toctree-l2"><a class="reference internal" href="../csv.html">Reading CSV files</a></li>
<li class="toctree-l2"><a class="reference internal" href="../feather.html">Feather File Format</a></li>
<li class="toctree-l2"><a class="reference internal" href="../json.html">Reading JSON files</a></li>
<li class="toctree-l2"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li>
<li class="toctree-l2"><a class="reference internal" href="../dataset.html">Tabular Datasets</a></li>
<li class="toctree-l2"><a class="reference internal" href="../cuda.html">CUDA Integration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extending_types.html">Extending pyarrow</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extending.html">Using pyarrow from C++ and Cython Code</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../api.html">API Reference</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Data Types and Schemas</a></li>
<li class="toctree-l3"><a class="reference internal" href="arrays.html">Arrays and Scalars</a></li>
<li class="toctree-l3"><a class="reference internal" href="memory.html">Buffers and Memory</a></li>
<li class="toctree-l3"><a class="reference internal" href="compute.html">Compute Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="files.html">Streams and File Access</a></li>
<li class="toctree-l3"><a class="reference internal" href="tables.html">Tables and Tensors</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">Serialization and IPC</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#inter-process-communication">Inter-Process Communication</a></li>
<li class="toctree-l4"><a class="reference internal" href="#serialization">Serialization</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="flight.html">Arrow Flight</a></li>
<li class="toctree-l3"><a class="reference internal" href="formats.html">Tabular File Formats</a></li>
<li class="toctree-l3"><a class="reference internal" href="filesystems.html">Filesystems</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset.html">Dataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="plasma.html">Plasma In-Memory Object Store</a></li>
<li class="toctree-l3"><a class="reference internal" href="cuda.html">CUDA Integration</a></li>
<li class="toctree-l3"><a class="reference internal" href="misc.html">Miscellaneous</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../getting_involved.html">Getting Involved</a></li>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks.html">Benchmarks</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/docs/r/">R</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/master/ruby/README.md">Ruby</a></li>
<li class="toctree-l1"><a class="reference external" href="https://docs.rs/crate/arrow/">Rust</a></li>
</ul>
<p class="caption"><span class="caption-text">Development</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../developers/contributing.html">Contributing to Apache Arrow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../developers/cpp/index.html">C++ Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../developers/python.html">Python Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../developers/archery.html">Daily Development using Archery</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../developers/crossbow.html">Packaging and Testing with Crossbow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../developers/docker.html">Running Docker Builds</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../developers/benchmarks.html">Benchmarks</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../developers/documentation.html">Building the Documentation</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">Apache Arrow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html" class="icon icon-home"></a> &raquo;</li>
<li><a href="../index.html">Python bindings</a> &raquo;</li>
<li><a href="../api.html">API Reference</a> &raquo;</li>
<li>Serialization and IPC</li>
<li class="wy-breadcrumbs-aside">
<a href="../../_sources/python/api/ipc.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="serialization-and-ipc">
<span id="api-ipc"></span><h1>Serialization and IPC<a class="headerlink" href="#serialization-and-ipc" title="Permalink to this headline"></a></h1>
<div class="section" id="inter-process-communication">
<h2>Inter-Process Communication<a class="headerlink" href="#inter-process-communication" title="Permalink to this headline"></a></h2>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.new_file.html#pyarrow.ipc.new_file" title="pyarrow.ipc.new_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.new_file</span></code></a>(sink, schema, *[, …])</p></td>
<td><p>Create an Arrow columnar IPC file writer instance</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.open_file.html#pyarrow.ipc.open_file" title="pyarrow.ipc.open_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.open_file</span></code></a>(source[, footer_offset])</p></td>
<td><p>Create reader for Arrow file format.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.new_stream.html#pyarrow.ipc.new_stream" title="pyarrow.ipc.new_stream"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.new_stream</span></code></a>(sink, schema, *[, …])</p></td>
<td><p>Create an Arrow columnar IPC stream writer instance</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.open_stream.html#pyarrow.ipc.open_stream" title="pyarrow.ipc.open_stream"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.open_stream</span></code></a>(source)</p></td>
<td><p>Create reader for Arrow streaming format.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.read_message.html#pyarrow.ipc.read_message" title="pyarrow.ipc.read_message"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.read_message</span></code></a>(source)</p></td>
<td><p>Read length-prefixed message from file or buffer-like object</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.read_record_batch.html#pyarrow.ipc.read_record_batch" title="pyarrow.ipc.read_record_batch"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.read_record_batch</span></code></a>(obj, Schema schema, …)</p></td>
<td><p>Read RecordBatch from message, given a known schema.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.get_record_batch_size.html#pyarrow.ipc.get_record_batch_size" title="pyarrow.ipc.get_record_batch_size"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.get_record_batch_size</span></code></a>(RecordBatch batch)</p></td>
<td><p>Return total size of serialized RecordBatch including metadata and padding.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.read_tensor.html#pyarrow.ipc.read_tensor" title="pyarrow.ipc.read_tensor"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.read_tensor</span></code></a>(source)</p></td>
<td><p>Read pyarrow.Tensor from pyarrow.NativeFile object from current position.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.write_tensor.html#pyarrow.ipc.write_tensor" title="pyarrow.ipc.write_tensor"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.write_tensor</span></code></a>(Tensor tensor, NativeFile dest)</p></td>
<td><p>Write pyarrow.Tensor to pyarrow.NativeFile object its current position.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.get_tensor_size.html#pyarrow.ipc.get_tensor_size" title="pyarrow.ipc.get_tensor_size"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.get_tensor_size</span></code></a>(Tensor tensor)</p></td>
<td><p>Return total size of serialized Tensor including metadata and padding.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.Message.html#pyarrow.ipc.Message" title="pyarrow.ipc.Message"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.Message</span></code></a>()</p></td>
<td><p>Container for an Arrow IPC message with metadata and optional body</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.MessageReader.html#pyarrow.ipc.MessageReader" title="pyarrow.ipc.MessageReader"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.MessageReader</span></code></a>()</p></td>
<td><p>Interface for reading Message objects from some source (like an InputStream)</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.RecordBatchFileReader.html#pyarrow.ipc.RecordBatchFileReader" title="pyarrow.ipc.RecordBatchFileReader"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.RecordBatchFileReader</span></code></a>(source[, …])</p></td>
<td><p>Class for reading Arrow record batch data from the Arrow binary file format</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.RecordBatchFileWriter.html#pyarrow.ipc.RecordBatchFileWriter" title="pyarrow.ipc.RecordBatchFileWriter"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.RecordBatchFileWriter</span></code></a>(sink, schema, *[, …])</p></td>
<td><p>Writer to create the Arrow binary file format</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.RecordBatchStreamReader.html#pyarrow.ipc.RecordBatchStreamReader" title="pyarrow.ipc.RecordBatchStreamReader"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.RecordBatchStreamReader</span></code></a>(source)</p></td>
<td><p>Reader for the Arrow streaming binary format.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.ipc.RecordBatchStreamWriter.html#pyarrow.ipc.RecordBatchStreamWriter" title="pyarrow.ipc.RecordBatchStreamWriter"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ipc.RecordBatchStreamWriter</span></code></a>(sink, schema, *)</p></td>
<td><p>Writer for the Arrow streaming binary format</p></td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="serialization">
<h2>Serialization<a class="headerlink" href="#serialization" title="Permalink to this headline"></a></h2>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>The serialization functionality is deprecated in pyarrow 2.0, and will
be removed in a future version. Use the standard library <code class="docutils literal notranslate"><span class="pre">pickle</span></code> or
the IPC functionality of pyarrow (see <a class="reference internal" href="../ipc.html#ipc"><span class="std std-ref">Streaming, Serialization, and IPC</span></a>).</p>
</div>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.serialize.html#pyarrow.serialize" title="pyarrow.serialize"><code class="xref py py-obj docutils literal notranslate"><span class="pre">serialize</span></code></a>(value, …)</p></td>
<td><p>EXPERIMENTAL: Serialize a general Python sequence for transient storage and transport.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.serialize_to.html#pyarrow.serialize_to" title="pyarrow.serialize_to"><code class="xref py py-obj docutils literal notranslate"><span class="pre">serialize_to</span></code></a>(value, sink, …)</p></td>
<td><p>EXPERIMENTAL: Serialize a Python sequence to a file.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.deserialize.html#pyarrow.deserialize" title="pyarrow.deserialize"><code class="xref py py-obj docutils literal notranslate"><span class="pre">deserialize</span></code></a>(obj, …)</p></td>
<td><p>EXPERIMENTAL: Deserialize Python object from Buffer or other Python object supporting the buffer protocol.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.deserialize_components.html#pyarrow.deserialize_components" title="pyarrow.deserialize_components"><code class="xref py py-obj docutils literal notranslate"><span class="pre">deserialize_components</span></code></a>(components, …)</p></td>
<td><p>Reconstruct Python object from output of SerializedPyObject.to_components.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.deserialize_from.html#pyarrow.deserialize_from" title="pyarrow.deserialize_from"><code class="xref py py-obj docutils literal notranslate"><span class="pre">deserialize_from</span></code></a>(source, base, …)</p></td>
<td><p>EXPERIMENTAL: Deserialize a Python sequence from a file.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.read_serialized.html#pyarrow.read_serialized" title="pyarrow.read_serialized"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_serialized</span></code></a>(source[, base])</p></td>
<td><p>EXPERIMENTAL: Read serialized Python sequence from file-like object.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../generated/pyarrow.SerializedPyObject.html#pyarrow.SerializedPyObject" title="pyarrow.SerializedPyObject"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SerializedPyObject</span></code></a></p></td>
<td><p>Arrow-serialized representation of Python object.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../generated/pyarrow.SerializationContext.html#pyarrow.SerializationContext" title="pyarrow.SerializationContext"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SerializationContext</span></code></a>()</p></td>
<td><p></p></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="../generated/pyarrow.ipc.new_file.html" class="btn btn-neutral float-right" title="pyarrow.ipc.new_file" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="../generated/pyarrow.Tensor.html" class="btn btn-neutral float-left" title="pyarrow.Tensor" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2016-2019 Apache Software Foundation
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a
<a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
<script type="text/javascript" src="/docs/_static/versionwarning.js"></script></body>
</html>