Google Git
Sign in
apache / arrow-site / asf-site / . / docs / 4.0 / python / plasma.html
blob: 5048e78e19f7f9cc28547202b4d2089010cc907a [file] [log] [blame]
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>The Plasma In-Memory Object Store &#8212; Apache Arrow v4.0.1</title>
<link href="../_static/css/theme.css" rel="stylesheet" />
<link href="../_static/css/index.c5995385ac14fb8791e8eb36b4908be2.css" rel="stylesheet" />
<link rel="stylesheet"
href="../_static/vendor/fontawesome/5.13.0/css/all.min.css">
<link rel="preload" as="font" type="font/woff2" crossorigin
href="../_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
<link rel="preload" as="font" type="font/woff2" crossorigin
href="../_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../_static/basic.css" />
<link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css" />
<link rel="preload" as="script" href="../_static/js/index.1c5a1a01449ed65a7b51.js">
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
<script src="../_static/jquery.js"></script>
<script src="../_static/underscore.js"></script>
<script src="../_static/doctools.js"></script>
<link rel="canonical" href="https://arrow.apache.org/docs/python/plasma.html" />
<link rel="shortcut icon" href="../_static/favicon.ico"/>
<link rel="canonical" href="https://arrow.apache.org/docs/python/plasma.html" />
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="NumPy Integration" href="numpy.html" />
<link rel="prev" title="pyarrow.HdfsFile" href="generated/pyarrow.HdfsFile.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en" />
</head>
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
<div class="container-fluid" id="banner"></div>
<div class="container-xl">
<div class="row">
<!-- Only show if we have sidebars configured, else just a small margin -->
<div class="col-12 col-md-3 bd-sidebar">
<a class="navbar-brand" href="../index.html">
<img src="../_static/arrow.png" class="logo" alt="logo">
</a>
<form class="bd-search d-flex align-items-center" action="../search.html" method="get">
<i class="icon fas fa-search"></i>
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" >
</form>
<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
<div class="bd-toc-item active">
<p class="caption">
<span class="caption-text">
Specifications and Protocols
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="../format/Versioning.html">
Format Versioning and Stability
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../format/Columnar.html">
Arrow Columnar Format
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../format/Flight.html">
Arrow Flight RPC
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../format/Integration.html">
Integration Testing
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../format/CDataInterface.html">
The Arrow C data interface
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../format/CStreamInterface.html">
The Arrow C stream interface
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../format/Other.html">
Other Data Structures
</a>
</li>
</ul>
<p class="caption">
<span class="caption-text">
Libraries
</span>
</p>
<ul class="current nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="../status.html">
Implementation Status
</a>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://arrow.apache.org/docs/c_glib/">
C/GLib
</a>
</li>
<li class="toctree-l1 has-children">
<a class="reference internal" href="../cpp/index.html">
C++
</a>
<input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/>
<label for="toctree-checkbox-1">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l2 has-children">
<a class="reference internal" href="../cpp/getting_started.html">
User Guide
</a>
<input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/>
<label for="toctree-checkbox-2">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/overview.html">
High-Level Overview
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/conventions.html">
Conventions
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/cmake.html">
Using Arrow C++ in your own project
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/memory.html">
Memory Management
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/arrays.html">
Arrays
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/datatypes.html">
Data Types
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/tables.html">
Tabular Data
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/compute.html">
Compute Functions
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/io.html">
Input / output and filesystems
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/ipc.html">
Reading and writing the Arrow IPC format
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/parquet.html">
Reading and writing Parquet files
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/csv.html">
Reading CSV files
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/json.html">
Reading JSON files
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/dataset.html">
Tabular Datasets
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/flight.html">
Arrow Flight RPC
</a>
</li>
</ul>
</li>
<li class="toctree-l2 has-children">
<a class="reference internal" href="../cpp/examples/index.html">
Examples
</a>
<input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/>
<label for="toctree-checkbox-3">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/examples/cmake_minimal_build.html">
Minimal build using CMake
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/examples/dataset_documentation_example.html">
Arrow Datasets example
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/examples/row_columnar_conversion.html">
Row to columnar conversion
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/examples/tuple_range_conversion.html">
std::tuple-like ranges to Arrow
</a>
</li>
</ul>
</li>
<li class="toctree-l2 has-children">
<a class="reference internal" href="../cpp/api.html">
API Reference
</a>
<input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/>
<label for="toctree-checkbox-4">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/support.html">
Programming Support
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/memory.html">
Memory (management)
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/datatype.html">
Data Types
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/array.html">
Arrays
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/scalar.html">
Scalars
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/builder.html">
Array Builders
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/table.html">
Two-dimensional Datasets
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/c_abi.html">
C Interfaces
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/compute.html">
Compute Functions
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/tensor.html">
Tensors
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/utilities.html">
Utilities
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/io.html">
Input / output
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/ipc.html">
Arrow IPC
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/formats.html">
File Formats
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/cuda.html">
CUDA support
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/flight.html">
Arrow Flight RPC
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/filesystem.html">
Filesystems
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="../cpp/api/dataset.html">
Dataset
</a>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow/blob/master/csharp/README.md">
C#
</a>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://godoc.org/github.com/apache/arrow/go/arrow">
Go
</a>
</li>
<li class="toctree-l1 has-children">
<a class="reference internal" href="../java/index.html">
Java
</a>
<input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/>
<label for="toctree-checkbox-5">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l2">
<a class="reference internal" href="../java/vector.html">
ValueVector
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="../java/vector_schema_root.html">
VectorSchemaRoot
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="../java/ipc.html">
Reading/Writing IPC formats
</a>
</li>
<li class="toctree-l2">
<a class="reference external" href="https://arrow.apache.org/docs/java/reference/">
Reference (javadoc)
</a>
</li>
</ul>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://arrow.apache.org/docs/js/">
JavaScript
</a>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow/blob/master/julia/Arrow/README.md">
Julia
</a>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow/blob/master/matlab/README.md">
MATLAB
</a>
</li>
<li class="toctree-l1 current active has-children">
<a class="reference internal" href="index.html">
Python
</a>
<input checked="" class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/>
<label for="toctree-checkbox-6">
<i class="fas fa-chevron-down">
</i>
</label>
<ul class="current">
<li class="toctree-l2">
<a class="reference internal" href="install.html">
Installing PyArrow
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="memory.html">
Memory and IO Interfaces
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="data.html">
Data Types and In-Memory Data Model
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="compute.html">
Compute Functions
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="ipc.html">
Streaming, Serialization, and IPC
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="filesystems.html">
Filesystem Interface
</a>
</li>
<li class="toctree-l2 has-children">
<a class="reference internal" href="filesystems_deprecated.html">
Filesystem Interface (legacy)
</a>
<input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/>
<label for="toctree-checkbox-7">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.hdfs.connect.html">
pyarrow.hdfs.connect
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.cat.html">
pyarrow.HadoopFileSystem.cat
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.chmod.html">
pyarrow.HadoopFileSystem.chmod
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.chown.html">
pyarrow.HadoopFileSystem.chown
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.delete.html">
pyarrow.HadoopFileSystem.delete
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.df.html">
pyarrow.HadoopFileSystem.df
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.disk_usage.html">
pyarrow.HadoopFileSystem.disk_usage
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.download.html">
pyarrow.HadoopFileSystem.download
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.exists.html">
pyarrow.HadoopFileSystem.exists
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.get_capacity.html">
pyarrow.HadoopFileSystem.get_capacity
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.get_space_used.html">
pyarrow.HadoopFileSystem.get_space_used
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.info.html">
pyarrow.HadoopFileSystem.info
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.ls.html">
pyarrow.HadoopFileSystem.ls
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.mkdir.html">
pyarrow.HadoopFileSystem.mkdir
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.open.html">
pyarrow.HadoopFileSystem.open
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.rename.html">
pyarrow.HadoopFileSystem.rename
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.rm.html">
pyarrow.HadoopFileSystem.rm
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HadoopFileSystem.upload.html">
pyarrow.HadoopFileSystem.upload
</a>
</li>
<li class="toctree-l3">
<a class="reference internal" href="generated/pyarrow.HdfsFile.html">
pyarrow.HdfsFile
</a>
</li>
</ul>
</li>
<li class="toctree-l2 current active">
<a class="current reference internal" href="#">
The Plasma In-Memory Object Store
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="numpy.html">
NumPy Integration
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="pandas.html">
Pandas Integration
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="timestamps.html">
Timestamps
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="csv.html">
Reading CSV files
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="feather.html">
Feather File Format
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="json.html">
Reading JSON files
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="parquet.html">
Reading and Writing the Apache Parquet Format
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="dataset.html">
Tabular Datasets
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="cuda.html">
CUDA Integration
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="extending_types.html">
Extending pyarrow
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="extending.html">
Using pyarrow from C++ and Cython Code
</a>
</li>
<li class="toctree-l2 has-children">
<a class="reference internal" href="api.html">
API Reference
</a>
<input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/>
<label for="toctree-checkbox-8">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/datatypes.html">
Data Types and Schemas
</a>
<input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/>
<label for="toctree-checkbox-9">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.null.html">
pyarrow.null
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.bool_.html">
pyarrow.bool_
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.int8.html">
pyarrow.int8
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.int16.html">
pyarrow.int16
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.int32.html">
pyarrow.int32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.int64.html">
pyarrow.int64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.uint8.html">
pyarrow.uint8
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.uint16.html">
pyarrow.uint16
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.uint32.html">
pyarrow.uint32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.uint64.html">
pyarrow.uint64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.float16.html">
pyarrow.float16
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.float32.html">
pyarrow.float32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.float64.html">
pyarrow.float64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.time32.html">
pyarrow.time32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.time64.html">
pyarrow.time64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.timestamp.html">
pyarrow.timestamp
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.date32.html">
pyarrow.date32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.date64.html">
pyarrow.date64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.binary.html">
pyarrow.binary
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.string.html">
pyarrow.string
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.utf8.html">
pyarrow.utf8
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.large_binary.html">
pyarrow.large_binary
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.large_string.html">
pyarrow.large_string
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.large_utf8.html">
pyarrow.large_utf8
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.decimal128.html">
pyarrow.decimal128
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.list_.html">
pyarrow.list_
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.large_list.html">
pyarrow.large_list
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.map_.html">
pyarrow.map_
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.struct.html">
pyarrow.struct
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dictionary.html">
pyarrow.dictionary
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.field.html">
pyarrow.field
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.schema.html">
pyarrow.schema
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.from_numpy_dtype.html">
pyarrow.from_numpy_dtype
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.DataType.html">
pyarrow.DataType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.DictionaryType.html">
pyarrow.DictionaryType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ListType.html">
pyarrow.ListType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.MapType.html">
pyarrow.MapType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.StructType.html">
pyarrow.StructType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UnionType.html">
pyarrow.UnionType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.TimestampType.html">
pyarrow.TimestampType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Time32Type.html">
pyarrow.Time32Type
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Time64Type.html">
pyarrow.Time64Type
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.FixedSizeBinaryType.html">
pyarrow.FixedSizeBinaryType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Decimal128Type.html">
pyarrow.Decimal128Type
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Field.html">
pyarrow.Field
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Schema.html">
pyarrow.Schema
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ExtensionType.html">
pyarrow.ExtensionType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.PyExtensionType.html">
pyarrow.PyExtensionType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.register_extension_type.html">
pyarrow.register_extension_type
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.unregister_extension_type.html">
pyarrow.unregister_extension_type
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_boolean.html">
pyarrow.types.is_boolean
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_integer.html">
pyarrow.types.is_integer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_signed_integer.html">
pyarrow.types.is_signed_integer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_unsigned_integer.html">
pyarrow.types.is_unsigned_integer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_int8.html">
pyarrow.types.is_int8
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_int16.html">
pyarrow.types.is_int16
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_int32.html">
pyarrow.types.is_int32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_int64.html">
pyarrow.types.is_int64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_uint8.html">
pyarrow.types.is_uint8
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_uint16.html">
pyarrow.types.is_uint16
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_uint32.html">
pyarrow.types.is_uint32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_uint64.html">
pyarrow.types.is_uint64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_floating.html">
pyarrow.types.is_floating
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_float16.html">
pyarrow.types.is_float16
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_float32.html">
pyarrow.types.is_float32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_float64.html">
pyarrow.types.is_float64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_decimal.html">
pyarrow.types.is_decimal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_list.html">
pyarrow.types.is_list
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_large_list.html">
pyarrow.types.is_large_list
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_struct.html">
pyarrow.types.is_struct
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_union.html">
pyarrow.types.is_union
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_nested.html">
pyarrow.types.is_nested
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_temporal.html">
pyarrow.types.is_temporal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_timestamp.html">
pyarrow.types.is_timestamp
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_date.html">
pyarrow.types.is_date
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_date32.html">
pyarrow.types.is_date32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_date64.html">
pyarrow.types.is_date64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_time.html">
pyarrow.types.is_time
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_time32.html">
pyarrow.types.is_time32
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_time64.html">
pyarrow.types.is_time64
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_null.html">
pyarrow.types.is_null
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_binary.html">
pyarrow.types.is_binary
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_unicode.html">
pyarrow.types.is_unicode
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_string.html">
pyarrow.types.is_string
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_large_binary.html">
pyarrow.types.is_large_binary
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_large_unicode.html">
pyarrow.types.is_large_unicode
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_large_string.html">
pyarrow.types.is_large_string
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_fixed_size_binary.html">
pyarrow.types.is_fixed_size_binary
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_map.html">
pyarrow.types.is_map
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.types.is_dictionary.html">
pyarrow.types.is_dictionary
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/arrays.html">
Arrays and Scalars
</a>
<input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/>
<label for="toctree-checkbox-10">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.array.html">
pyarrow.array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.nulls.html">
pyarrow.nulls
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Array.html">
pyarrow.Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.BooleanArray.html">
pyarrow.BooleanArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.FloatingPointArray.html">
pyarrow.FloatingPointArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.IntegerArray.html">
pyarrow.IntegerArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int8Array.html">
pyarrow.Int8Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int16Array.html">
pyarrow.Int16Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int32Array.html">
pyarrow.Int32Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int64Array.html">
pyarrow.Int64Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.NullArray.html">
pyarrow.NullArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.NumericArray.html">
pyarrow.NumericArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt8Array.html">
pyarrow.UInt8Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt16Array.html">
pyarrow.UInt16Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt32Array.html">
pyarrow.UInt32Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt64Array.html">
pyarrow.UInt64Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.BinaryArray.html">
pyarrow.BinaryArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.StringArray.html">
pyarrow.StringArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.FixedSizeBinaryArray.html">
pyarrow.FixedSizeBinaryArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.LargeBinaryArray.html">
pyarrow.LargeBinaryArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.LargeStringArray.html">
pyarrow.LargeStringArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Time32Array.html">
pyarrow.Time32Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Time64Array.html">
pyarrow.Time64Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Date32Array.html">
pyarrow.Date32Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Date64Array.html">
pyarrow.Date64Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.TimestampArray.html">
pyarrow.TimestampArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Decimal128Array.html">
pyarrow.Decimal128Array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.DictionaryArray.html">
pyarrow.DictionaryArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ListArray.html">
pyarrow.ListArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.LargeListArray.html">
pyarrow.LargeListArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.StructArray.html">
pyarrow.StructArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UnionArray.html">
pyarrow.UnionArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ExtensionArray.html">
pyarrow.ExtensionArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.scalar.html">
pyarrow.scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.NA.html">
pyarrow.NA
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Scalar.html">
pyarrow.Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.BooleanScalar.html">
pyarrow.BooleanScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int8Scalar.html">
pyarrow.Int8Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int16Scalar.html">
pyarrow.Int16Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int32Scalar.html">
pyarrow.Int32Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Int64Scalar.html">
pyarrow.Int64Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt8Scalar.html">
pyarrow.UInt8Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt16Scalar.html">
pyarrow.UInt16Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt32Scalar.html">
pyarrow.UInt32Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UInt64Scalar.html">
pyarrow.UInt64Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.FloatScalar.html">
pyarrow.FloatScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.DoubleScalar.html">
pyarrow.DoubleScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.BinaryScalar.html">
pyarrow.BinaryScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.StringScalar.html">
pyarrow.StringScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.FixedSizeBinaryScalar.html">
pyarrow.FixedSizeBinaryScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.LargeBinaryScalar.html">
pyarrow.LargeBinaryScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.LargeStringScalar.html">
pyarrow.LargeStringScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Time32Scalar.html">
pyarrow.Time32Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Time64Scalar.html">
pyarrow.Time64Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Date32Scalar.html">
pyarrow.Date32Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Date64Scalar.html">
pyarrow.Date64Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.TimestampScalar.html">
pyarrow.TimestampScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Decimal128Scalar.html">
pyarrow.Decimal128Scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.DictionaryScalar.html">
pyarrow.DictionaryScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ListScalar.html">
pyarrow.ListScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.LargeListScalar.html">
pyarrow.LargeListScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.StructScalar.html">
pyarrow.StructScalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.UnionScalar.html">
pyarrow.UnionScalar
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/memory.html">
Buffers and Memory
</a>
<input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" type="checkbox"/>
<label for="toctree-checkbox-11">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.allocate_buffer.html">
pyarrow.allocate_buffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.py_buffer.html">
pyarrow.py_buffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.foreign_buffer.html">
pyarrow.foreign_buffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Buffer.html">
pyarrow.Buffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ResizableBuffer.html">
pyarrow.ResizableBuffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compress.html">
pyarrow.compress
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.decompress.html">
pyarrow.decompress
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.MemoryPool.html">
pyarrow.MemoryPool
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.default_memory_pool.html">
pyarrow.default_memory_pool
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.jemalloc_memory_pool.html">
pyarrow.jemalloc_memory_pool
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.mimalloc_memory_pool.html">
pyarrow.mimalloc_memory_pool
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.system_memory_pool.html">
pyarrow.system_memory_pool
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.jemalloc_set_decay_ms.html">
pyarrow.jemalloc_set_decay_ms
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.set_memory_pool.html">
pyarrow.set_memory_pool
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.log_memory_allocations.html">
pyarrow.log_memory_allocations
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.total_allocated_bytes.html">
pyarrow.total_allocated_bytes
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/compute.html">
Compute Functions
</a>
<input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" type="checkbox"/>
<label for="toctree-checkbox-12">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.count.html">
pyarrow.compute.count
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.mean.html">
pyarrow.compute.mean
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.min_max.html">
pyarrow.compute.min_max
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.mode.html">
pyarrow.compute.mode
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.stddev.html">
pyarrow.compute.stddev
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.sum.html">
pyarrow.compute.sum
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.variance.html">
pyarrow.compute.variance
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.add.html">
pyarrow.compute.add
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.add_checked.html">
pyarrow.compute.add_checked
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.divide.html">
pyarrow.compute.divide
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.divide_checked.html">
pyarrow.compute.divide_checked
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.multiply.html">
pyarrow.compute.multiply
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.multiply_checked.html">
pyarrow.compute.multiply_checked
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.subtract.html">
pyarrow.compute.subtract
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.subtract_checked.html">
pyarrow.compute.subtract_checked
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.power.html">
pyarrow.compute.power
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.power_checked.html">
pyarrow.compute.power_checked
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.equal.html">
pyarrow.compute.equal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.greater.html">
pyarrow.compute.greater
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.greater_equal.html">
pyarrow.compute.greater_equal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.less.html">
pyarrow.compute.less
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.less_equal.html">
pyarrow.compute.less_equal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.not_equal.html">
pyarrow.compute.not_equal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.and_.html">
pyarrow.compute.and_
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.and_kleene.html">
pyarrow.compute.and_kleene
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.all.html">
pyarrow.compute.all
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.any.html">
pyarrow.compute.any
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.invert.html">
pyarrow.compute.invert
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.or_.html">
pyarrow.compute.or_
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.or_kleene.html">
pyarrow.compute.or_kleene
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.xor.html">
pyarrow.compute.xor
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_alnum.html">
pyarrow.compute.ascii_is_alnum
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_alpha.html">
pyarrow.compute.ascii_is_alpha
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_decimal.html">
pyarrow.compute.ascii_is_decimal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_lower.html">
pyarrow.compute.ascii_is_lower
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_printable.html">
pyarrow.compute.ascii_is_printable
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_space.html">
pyarrow.compute.ascii_is_space
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_upper.html">
pyarrow.compute.ascii_is_upper
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_alnum.html">
pyarrow.compute.utf8_is_alnum
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_alpha.html">
pyarrow.compute.utf8_is_alpha
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_decimal.html">
pyarrow.compute.utf8_is_decimal
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_digit.html">
pyarrow.compute.utf8_is_digit
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_lower.html">
pyarrow.compute.utf8_is_lower
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_numeric.html">
pyarrow.compute.utf8_is_numeric
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_printable.html">
pyarrow.compute.utf8_is_printable
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_space.html">
pyarrow.compute.utf8_is_space
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_upper.html">
pyarrow.compute.utf8_is_upper
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_is_title.html">
pyarrow.compute.ascii_is_title
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_is_title.html">
pyarrow.compute.utf8_is_title
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.string_is_ascii.html">
pyarrow.compute.string_is_ascii
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_lower.html">
pyarrow.compute.ascii_lower
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.ascii_upper.html">
pyarrow.compute.ascii_upper
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_lower.html">
pyarrow.compute.utf8_lower
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.utf8_upper.html">
pyarrow.compute.utf8_upper
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.index_in.html">
pyarrow.compute.index_in
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.is_in.html">
pyarrow.compute.is_in
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.match_substring.html">
pyarrow.compute.match_substring
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.match_substring_regex.html">
pyarrow.compute.match_substring_regex
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.cast.html">
pyarrow.compute.cast
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.strptime.html">
pyarrow.compute.strptime
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.filter.html">
pyarrow.compute.filter
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.take.html">
pyarrow.compute.take
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.dictionary_encode.html">
pyarrow.compute.dictionary_encode
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.unique.html">
pyarrow.compute.unique
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.value_counts.html">
pyarrow.compute.value_counts
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.partition_nth_indices.html">
pyarrow.compute.partition_nth_indices
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.sort_indices.html">
pyarrow.compute.sort_indices
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.binary_length.html">
pyarrow.compute.binary_length
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.fill_null.html">
pyarrow.compute.fill_null
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.is_null.html">
pyarrow.compute.is_null
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.is_valid.html">
pyarrow.compute.is_valid
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.list_value_length.html">
pyarrow.compute.list_value_length
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.list_flatten.html">
pyarrow.compute.list_flatten
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.compute.list_parent_indices.html">
pyarrow.compute.list_parent_indices
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/files.html">
Streams and File Access
</a>
<input class="toctree-checkbox" id="toctree-checkbox-13" name="toctree-checkbox-13" type="checkbox"/>
<label for="toctree-checkbox-13">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.input_stream.html">
pyarrow.input_stream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.output_stream.html">
pyarrow.output_stream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.memory_map.html">
pyarrow.memory_map
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.create_memory_map.html">
pyarrow.create_memory_map
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.NativeFile.html">
pyarrow.NativeFile
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.OSFile.html">
pyarrow.OSFile
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.PythonFile.html">
pyarrow.PythonFile
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.BufferReader.html">
pyarrow.BufferReader
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.BufferOutputStream.html">
pyarrow.BufferOutputStream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.FixedSizeBufferWriter.html">
pyarrow.FixedSizeBufferWriter
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.MemoryMappedFile.html">
pyarrow.MemoryMappedFile
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.CompressedInputStream.html">
pyarrow.CompressedInputStream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.CompressedOutputStream.html">
pyarrow.CompressedOutputStream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.hdfs.connect.html">
pyarrow.hdfs.connect
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.LocalFileSystem.html">
pyarrow.LocalFileSystem
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/tables.html">
Tables and Tensors
</a>
<input class="toctree-checkbox" id="toctree-checkbox-14" name="toctree-checkbox-14" type="checkbox"/>
<label for="toctree-checkbox-14">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.chunked_array.html">
pyarrow.chunked_array
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.concat_arrays.html">
pyarrow.concat_arrays
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.concat_tables.html">
pyarrow.concat_tables
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.record_batch.html">
pyarrow.record_batch
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.table.html">
pyarrow.table
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ChunkedArray.html">
pyarrow.ChunkedArray
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.RecordBatch.html">
pyarrow.RecordBatch
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Table.html">
pyarrow.Table
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.Tensor.html">
pyarrow.Tensor
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/ipc.html">
Serialization and IPC
</a>
<input class="toctree-checkbox" id="toctree-checkbox-15" name="toctree-checkbox-15" type="checkbox"/>
<label for="toctree-checkbox-15">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.new_file.html">
pyarrow.ipc.new_file
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.open_file.html">
pyarrow.ipc.open_file
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.new_stream.html">
pyarrow.ipc.new_stream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.open_stream.html">
pyarrow.ipc.open_stream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.read_message.html">
pyarrow.ipc.read_message
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.read_record_batch.html">
pyarrow.ipc.read_record_batch
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.get_record_batch_size.html">
pyarrow.ipc.get_record_batch_size
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.read_tensor.html">
pyarrow.ipc.read_tensor
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.write_tensor.html">
pyarrow.ipc.write_tensor
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.get_tensor_size.html">
pyarrow.ipc.get_tensor_size
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.Message.html">
pyarrow.ipc.Message
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.MessageReader.html">
pyarrow.ipc.MessageReader
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.RecordBatchFileReader.html">
pyarrow.ipc.RecordBatchFileReader
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.RecordBatchFileWriter.html">
pyarrow.ipc.RecordBatchFileWriter
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.RecordBatchStreamReader.html">
pyarrow.ipc.RecordBatchStreamReader
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.ipc.RecordBatchStreamWriter.html">
pyarrow.ipc.RecordBatchStreamWriter
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.serialize.html">
pyarrow.serialize
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.serialize_to.html">
pyarrow.serialize_to
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.deserialize.html">
pyarrow.deserialize
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.deserialize_components.html">
pyarrow.deserialize_components
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.deserialize_from.html">
pyarrow.deserialize_from
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.read_serialized.html">
pyarrow.read_serialized
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.SerializedPyObject.html">
pyarrow.SerializedPyObject
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.SerializationContext.html">
pyarrow.SerializationContext
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/flight.html">
Arrow Flight
</a>
<input class="toctree-checkbox" id="toctree-checkbox-16" name="toctree-checkbox-16" type="checkbox"/>
<label for="toctree-checkbox-16">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.Action.html">
pyarrow.flight.Action
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.ActionType.html">
pyarrow.flight.ActionType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.DescriptorType.html">
pyarrow.flight.DescriptorType
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.FlightDescriptor.html">
pyarrow.flight.FlightDescriptor
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.FlightEndpoint.html">
pyarrow.flight.FlightEndpoint
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.FlightInfo.html">
pyarrow.flight.FlightInfo
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.Location.html">
pyarrow.flight.Location
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.Ticket.html">
pyarrow.flight.Ticket
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.Result.html">
pyarrow.flight.Result
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.FlightCallOptions.html">
pyarrow.flight.FlightCallOptions
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.FlightClient.html">
pyarrow.flight.FlightClient
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.ClientMiddlewareFactory.html">
pyarrow.flight.ClientMiddlewareFactory
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.ClientMiddleware.html">
pyarrow.flight.ClientMiddleware
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.FlightServerBase.html">
pyarrow.flight.FlightServerBase
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.GeneratorStream.html">
pyarrow.flight.GeneratorStream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.RecordBatchStream.html">
pyarrow.flight.RecordBatchStream
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.ServerMiddlewareFactory.html">
pyarrow.flight.ServerMiddlewareFactory
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.ServerMiddleware.html">
pyarrow.flight.ServerMiddleware
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.ClientAuthHandler.html">
pyarrow.flight.ClientAuthHandler
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.ServerAuthHandler.html">
pyarrow.flight.ServerAuthHandler
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.FlightMethod.html">
pyarrow.flight.FlightMethod
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.flight.CallInfo.html">
pyarrow.flight.CallInfo
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/formats.html">
Tabular File Formats
</a>
<input class="toctree-checkbox" id="toctree-checkbox-17" name="toctree-checkbox-17" type="checkbox"/>
<label for="toctree-checkbox-17">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.csv.ReadOptions.html">
pyarrow.csv.ReadOptions
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.csv.ParseOptions.html">
pyarrow.csv.ParseOptions
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.csv.ConvertOptions.html">
pyarrow.csv.ConvertOptions
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.csv.read_csv.html">
pyarrow.csv.read_csv
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.csv.open_csv.html">
pyarrow.csv.open_csv
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.csv.CSVStreamingReader.html">
pyarrow.csv.CSVStreamingReader
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.feather.read_feather.html">
pyarrow.feather.read_feather
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.feather.read_table.html">
pyarrow.feather.read_table
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.feather.write_feather.html">
pyarrow.feather.write_feather
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.json.ReadOptions.html">
pyarrow.json.ReadOptions
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.json.ParseOptions.html">
pyarrow.json.ParseOptions
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.json.read_json.html">
pyarrow.json.read_json
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.ParquetDataset.html">
pyarrow.parquet.ParquetDataset
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.ParquetFile.html">
pyarrow.parquet.ParquetFile
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.ParquetWriter.html">
pyarrow.parquet.ParquetWriter
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.read_table.html">
pyarrow.parquet.read_table
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.read_metadata.html">
pyarrow.parquet.read_metadata
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.read_pandas.html">
pyarrow.parquet.read_pandas
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.read_schema.html">
pyarrow.parquet.read_schema
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.write_metadata.html">
pyarrow.parquet.write_metadata
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.write_table.html">
pyarrow.parquet.write_table
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.parquet.write_to_dataset.html">
pyarrow.parquet.write_to_dataset
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.orc.ORCFile.html">
pyarrow.orc.ORCFile
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/filesystems.html">
Filesystems
</a>
<input class="toctree-checkbox" id="toctree-checkbox-18" name="toctree-checkbox-18" type="checkbox"/>
<label for="toctree-checkbox-18">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.FileInfo.html">
pyarrow.fs.FileInfo
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.FileSelector.html">
pyarrow.fs.FileSelector
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.FileSystem.html">
pyarrow.fs.FileSystem
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.LocalFileSystem.html">
pyarrow.fs.LocalFileSystem
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.S3FileSystem.html">
pyarrow.fs.S3FileSystem
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.HadoopFileSystem.html">
pyarrow.fs.HadoopFileSystem
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.SubTreeFileSystem.html">
pyarrow.fs.SubTreeFileSystem
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.PyFileSystem.html">
pyarrow.fs.PyFileSystem
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.FileSystemHandler.html">
pyarrow.fs.FileSystemHandler
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.fs.FSSpecHandler.html">
pyarrow.fs.FSSpecHandler
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/dataset.html">
Dataset
</a>
<input class="toctree-checkbox" id="toctree-checkbox-19" name="toctree-checkbox-19" type="checkbox"/>
<label for="toctree-checkbox-19">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.dataset.html">
pyarrow.dataset.dataset
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.parquet_dataset.html">
pyarrow.dataset.parquet_dataset
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.partitioning.html">
pyarrow.dataset.partitioning
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.field.html">
pyarrow.dataset.field
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.scalar.html">
pyarrow.dataset.scalar
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.FileFormat.html">
pyarrow.dataset.FileFormat
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.ParquetFileFormat.html">
pyarrow.dataset.ParquetFileFormat
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.Partitioning.html">
pyarrow.dataset.Partitioning
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.PartitioningFactory.html">
pyarrow.dataset.PartitioningFactory
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.DirectoryPartitioning.html">
pyarrow.dataset.DirectoryPartitioning
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.HivePartitioning.html">
pyarrow.dataset.HivePartitioning
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.Dataset.html">
pyarrow.dataset.Dataset
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.FileSystemDataset.html">
pyarrow.dataset.FileSystemDataset
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.FileSystemFactoryOptions.html">
pyarrow.dataset.FileSystemFactoryOptions
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.FileSystemDatasetFactory.html">
pyarrow.dataset.FileSystemDatasetFactory
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.UnionDataset.html">
pyarrow.dataset.UnionDataset
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.Scanner.html">
pyarrow.dataset.Scanner
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.dataset.Expression.html">
pyarrow.dataset.Expression
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/plasma.html">
Plasma In-Memory Object Store
</a>
<input class="toctree-checkbox" id="toctree-checkbox-20" name="toctree-checkbox-20" type="checkbox"/>
<label for="toctree-checkbox-20">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.plasma.ObjectID.html">
pyarrow.plasma.ObjectID
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.plasma.PlasmaClient.html">
pyarrow.plasma.PlasmaClient
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.plasma.PlasmaBuffer.html">
pyarrow.plasma.PlasmaBuffer
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/cuda.html">
CUDA Integration
</a>
<input class="toctree-checkbox" id="toctree-checkbox-21" name="toctree-checkbox-21" type="checkbox"/>
<label for="toctree-checkbox-21">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.Context.html">
pyarrow.cuda.Context
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.CudaBuffer.html">
pyarrow.cuda.CudaBuffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.new_host_buffer.html">
pyarrow.cuda.new_host_buffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.HostBuffer.html">
pyarrow.cuda.HostBuffer
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.BufferReader.html">
pyarrow.cuda.BufferReader
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.BufferWriter.html">
pyarrow.cuda.BufferWriter
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.serialize_record_batch.html">
pyarrow.cuda.serialize_record_batch
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.read_record_batch.html">
pyarrow.cuda.read_record_batch
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.read_message.html">
pyarrow.cuda.read_message
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cuda.IpcMemHandle.html">
pyarrow.cuda.IpcMemHandle
</a>
</li>
</ul>
</li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="api/misc.html">
Miscellaneous
</a>
<input class="toctree-checkbox" id="toctree-checkbox-22" name="toctree-checkbox-22" type="checkbox"/>
<label for="toctree-checkbox-22">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.cpu_count.html">
pyarrow.cpu_count
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.set_cpu_count.html">
pyarrow.set_cpu_count
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.get_include.html">
pyarrow.get_include
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.get_libraries.html">
pyarrow.get_libraries
</a>
</li>
<li class="toctree-l4">
<a class="reference internal" href="generated/pyarrow.get_library_dirs.html">
pyarrow.get_library_dirs
</a>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2">
<a class="reference internal" href="getting_involved.html">
Getting Involved
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="benchmarks.html">
Benchmarks
</a>
</li>
</ul>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://arrow.apache.org/docs/r/">
R
</a>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://github.com/apache/arrow/blob/master/ruby/README.md">
Ruby
</a>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://docs.rs/crate/arrow/">
Rust
</a>
</li>
</ul>
<p class="caption">
<span class="caption-text">
Development
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="../developers/contributing.html">
Contributing to Apache Arrow
</a>
</li>
<li class="toctree-l1 has-children">
<a class="reference internal" href="../developers/cpp/index.html">
C++ Development
</a>
<input class="toctree-checkbox" id="toctree-checkbox-23" name="toctree-checkbox-23" type="checkbox"/>
<label for="toctree-checkbox-23">
<i class="fas fa-chevron-down">
</i>
</label>
<ul>
<li class="toctree-l2">
<a class="reference internal" href="../developers/cpp/building.html">
Building Arrow C++
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="../developers/cpp/development.html">
Development Guidelines
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="../developers/cpp/windows.html">
Developing on Windows
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="../developers/cpp/conventions.html">
Conventions
</a>
</li>
<li class="toctree-l2">
<a class="reference internal" href="../developers/cpp/fuzzing.html">
Fuzzing Arrow C++
</a>
</li>
</ul>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../developers/python.html">
Python Development
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../developers/archery.html">
Daily Development using Archery
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../developers/crossbow.html">
Packaging and Testing with Crossbow
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../developers/docker.html">
Running Docker Builds
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../developers/benchmarks.html">
Benchmarks
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="../developers/documentation.html">
Building the Documentation
</a>
</li>
</ul>
</div>
</nav>
</div>
<div class="d-none d-xl-block col-xl-2 bd-toc">
<div class="toc-item">
<div class="tocsection onthispage pt-5 pb-3">
<i class="fas fa-list"></i> On this page
</div>
<nav id="bd-toc-nav">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#the-plasma-api">
The Plasma API
</a>
<ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#starting-the-plasma-store">
Starting the Plasma store
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#creating-a-plasma-client">
Creating a Plasma client
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#object-ids">
Object IDs
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#putting-and-getting-python-objects">
Putting and Getting Python Objects
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#creating-an-object-buffer">
Creating an Object Buffer
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#getting-an-object-buffer">
Getting an Object Buffer
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#listing-objects-in-the-store">
Listing objects in the store
</a>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#using-arrow-and-pandas-with-plasma">
Using Arrow and Pandas with Plasma
</a>
<ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#storing-arrow-objects-in-plasma">
Storing Arrow Objects in Plasma
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#getting-arrow-objects-from-plasma">
Getting Arrow Objects from Plasma
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#storing-pandas-dataframes-in-plasma">
Storing Pandas DataFrames in Plasma
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#getting-pandas-dataframes-from-plasma">
Getting Pandas DataFrames from Plasma
</a>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#using-plasma-with-huge-pages">
Using Plasma with Huge Pages
</a>
</li>
</ul>
</nav>
</div>
<div class="toc-item">
</div>
</div>
<main class="col-12 col-md-9 col-xl-7 py-md-5 pl-md-5 pr-md-4 bd-content" role="main">
<div>
<div class="section" id="the-plasma-in-memory-object-store">
<span id="plasma"></span><h1>The Plasma In-Memory Object Store<a class="headerlink" href="#the-plasma-in-memory-object-store" title="Permalink to this headline"></a></h1>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>As present, Plasma is only supported for use on Linux and macOS.</p>
</div>
<div class="section" id="the-plasma-api">
<h2>The Plasma API<a class="headerlink" href="#the-plasma-api" title="Permalink to this headline"></a></h2>
<div class="section" id="starting-the-plasma-store">
<h3>Starting the Plasma store<a class="headerlink" href="#starting-the-plasma-store" title="Permalink to this headline"></a></h3>
<p>You can start the Plasma store by issuing a terminal command similar to the
following:</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>plasma_store -m <span class="m">1000000000</span> -s /tmp/plasma
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">-m</span></code> flag specifies the size of the store in bytes, and the <code class="docutils literal notranslate"><span class="pre">-s</span></code> flag
specifies the socket that the store will listen at. Thus, the above command
allows the Plasma store to use up to 1GB of memory, and sets the socket to
<code class="docutils literal notranslate"><span class="pre">/tmp/plasma</span></code>.</p>
<p>Leaving the current terminal window open as long as Plasma store should keep
running. Messages, concerning such as disconnecting clients, may occasionally be
printed to the screen. To stop running the Plasma store, you can press
<code class="docutils literal notranslate"><span class="pre">Ctrl-C</span></code> in the terminal.</p>
</div>
<div class="section" id="creating-a-plasma-client">
<h3>Creating a Plasma client<a class="headerlink" href="#creating-a-plasma-client" title="Permalink to this headline"></a></h3>
<p>To start a Plasma client from Python, call <code class="docutils literal notranslate"><span class="pre">plasma.connect</span></code> using the same
socket name:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pyarrow.plasma</span> <span class="kn">as</span> <span class="nn">plasma</span>
<span class="n">client</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="s2">&quot;/tmp/plasma&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>If the following error occurs from running the above Python code, that
means that either the socket given is incorrect, or the <code class="docutils literal notranslate"><span class="pre">./plasma_store</span></code> is
not currently running. Check to see if the Plasma store is still running.</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>&gt;&gt;&gt; <span class="nv">client</span> <span class="o">=</span> plasma.connect<span class="o">(</span><span class="s2">&quot;/tmp/plasma&quot;</span><span class="o">)</span>
Connection to socket failed <span class="k">for</span> pathname /tmp/plasma
Could not connect to socket /tmp/plasma
</pre></div>
</div>
</div>
<div class="section" id="object-ids">
<h3>Object IDs<a class="headerlink" href="#object-ids" title="Permalink to this headline"></a></h3>
<p>Each object in the Plasma store should be associated with a unique ID. The
Object ID then serves as a key that any client can use to retrieve that object
from the Plasma store. You can form an <code class="docutils literal notranslate"><span class="pre">ObjectID</span></code> object from a byte string of
length 20.</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="c1"># Create an ObjectID.</span>
&gt;&gt;&gt; <span class="nv">id</span> <span class="o">=</span> plasma.ObjectID<span class="o">(</span><span class="m">20</span> * b<span class="s2">&quot;a&quot;</span><span class="o">)</span>
<span class="c1"># The character &quot;a&quot; is encoded as 61 in hex.</span>
&gt;&gt;&gt; id
ObjectID<span class="o">(</span><span class="m">6161616161616161616161616161616161616161</span><span class="o">)</span>
</pre></div>
</div>
<p>The random generation of Object IDs is often good enough to ensure unique IDs.
You can easily create a helper function that randomly generates object IDs as
follows:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="k">def</span> <span class="nf">random_object_id</span><span class="p">():</span>
<span class="k">return</span> <span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">bytes</span><span class="p">(</span><span class="mi">20</span><span class="p">))</span>
</pre></div>
</div>
</div>
<div class="section" id="putting-and-getting-python-objects">
<h3>Putting and Getting Python Objects<a class="headerlink" href="#putting-and-getting-python-objects" title="Permalink to this headline"></a></h3>
<p>Plasma supports two APIs for creating and accessing objects: A high level
API that allows storing and retrieving Python objects and a low level
API that allows creating, writing and sealing buffers and operating on
the binary data directly. In this section we describe the high level API.</p>
<p>This is how you can put and get a Python object:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Create a python object.</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="s2">&quot;hello, world&quot;</span><span class="p">)</span>
<span class="c1"># Get the object.</span>
<span class="n">client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">object_id</span><span class="p">)</span>
</pre></div>
</div>
<p>This works with all Python objects supported by the Arrow Python object
serialization.</p>
<p>You can also get multiple objects at the same time (which can be more
efficient since it avoids IPC round trips):</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Create multiple python objects.</span>
<span class="n">object_id1</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">object_id2</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="n">object_id3</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
<span class="c1"># Get the objects.</span>
<span class="n">client</span><span class="o">.</span><span class="n">get</span><span class="p">([</span><span class="n">object_id1</span><span class="p">,</span> <span class="n">object_id2</span><span class="p">,</span> <span class="n">object_id3</span><span class="p">])</span>
</pre></div>
</div>
<p>Furthermore, it is possible to provide a timeout for the get call. If the
object is not available within the timeout, the special object
<cite>pyarrow.ObjectNotAvailable</cite> will be returned.</p>
</div>
<div class="section" id="creating-an-object-buffer">
<h3>Creating an Object Buffer<a class="headerlink" href="#creating-an-object-buffer" title="Permalink to this headline"></a></h3>
<p>Objects are created in Plasma in two stages. First, they are <strong>created</strong>, which
allocates a buffer for the object. At this point, the client can write to the
buffer and construct the object within the allocated buffer.</p>
<p>To create an object for Plasma, you need to create an object ID, as well as
give the object’s maximum size in bytes.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Create an object buffer.</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="mi">20</span> <span class="o">*</span> <span class="sa">b</span><span class="s2">&quot;a&quot;</span><span class="p">)</span>
<span class="n">object_size</span> <span class="o">=</span> <span class="mi">1000</span>
<span class="nb">buffer</span> <span class="o">=</span> <span class="n">memoryview</span><span class="p">(</span><span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">object_id</span><span class="p">,</span> <span class="n">object_size</span><span class="p">))</span>
<span class="c1"># Write to the buffer.</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1000</span><span class="p">):</span>
<span class="nb">buffer</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">i</span> <span class="o">%</span> <span class="mi">128</span>
</pre></div>
</div>
<p>When the client is done, the client <strong>seals</strong> the buffer, making the object
immutable, and making it available to other Plasma clients.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Seal the object. This makes the object immutable and available to other clients.</span>
<span class="n">client</span><span class="o">.</span><span class="n">seal</span><span class="p">(</span><span class="n">object_id</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="getting-an-object-buffer">
<h3>Getting an Object Buffer<a class="headerlink" href="#getting-an-object-buffer" title="Permalink to this headline"></a></h3>
<p>After an object has been sealed, any client who knows the object ID can get
the object buffer.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Create a different client. Note that this second client could be</span>
<span class="c1"># created in the same or in a separate, concurrent Python session.</span>
<span class="n">client2</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="s2">&quot;/tmp/plasma&quot;</span><span class="p">)</span>
<span class="c1"># Get the object in the second client. This blocks until the object has been sealed.</span>
<span class="n">object_id2</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="mi">20</span> <span class="o">*</span> <span class="sa">b</span><span class="s2">&quot;a&quot;</span><span class="p">)</span>
<span class="p">[</span><span class="n">buffer2</span><span class="p">]</span> <span class="o">=</span> <span class="n">client2</span><span class="o">.</span><span class="n">get_buffers</span><span class="p">([</span><span class="n">object_id</span><span class="p">])</span>
</pre></div>
</div>
<p>If the object has not been sealed yet, then the call to client.get_buffers will
block until the object has been sealed by the client constructing the object.
Using the <code class="docutils literal notranslate"><span class="pre">timeout_ms</span></code> argument to get, you can specify a timeout for this (in
milliseconds). After the timeout, the interpreter will yield control back.</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>&gt;&gt;&gt; buffer
&lt;memory at 0x7fdbdc96e708&gt;
&gt;&gt;&gt; buffer<span class="o">[</span><span class="m">1</span><span class="o">]</span>
<span class="m">1</span>
&gt;&gt;&gt; buffer2
&lt;plasma.plasma.PlasmaBuffer object at 0x7fdbf2770e88&gt;
&gt;&gt;&gt; <span class="nv">view2</span> <span class="o">=</span> memoryview<span class="o">(</span>buffer2<span class="o">)</span>
&gt;&gt;&gt; view2<span class="o">[</span><span class="m">1</span><span class="o">]</span>
<span class="m">1</span>
&gt;&gt;&gt; view2<span class="o">[</span><span class="m">129</span><span class="o">]</span>
<span class="m">1</span>
&gt;&gt;&gt; bytes<span class="o">(</span>buffer<span class="o">[</span><span class="m">1</span>:4<span class="o">])</span>
b<span class="s1">&#39;\x01\x02\x03&#39;</span>
&gt;&gt;&gt; bytes<span class="o">(</span>view2<span class="o">[</span><span class="m">1</span>:4<span class="o">])</span>
b<span class="s1">&#39;\x01\x02\x03&#39;</span>
</pre></div>
</div>
</div>
<div class="section" id="listing-objects-in-the-store">
<h3>Listing objects in the store<a class="headerlink" href="#listing-objects-in-the-store" title="Permalink to this headline"></a></h3>
<p>The objects in the store can be listed in the following way (note that
this functionality is currently experimental and the concrete representation
of the object info might change in the future):</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pyarrow.plasma</span> <span class="kn">as</span> <span class="nn">plasma</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="n">client</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="s2">&quot;/tmp/plasma&quot;</span><span class="p">)</span>
<span class="n">client</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="s2">&quot;hello, world&quot;</span><span class="p">)</span>
<span class="c1"># Sleep a little so we get different creation times</span>
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="n">client</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="s2">&quot;another object&quot;</span><span class="p">)</span>
<span class="c1"># Create an object that is not sealed yet</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="o">.</span><span class="n">from_random</span><span class="p">()</span>
<span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">object_id</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
<span class="k">print</span><span class="p">(</span><span class="n">client</span><span class="o">.</span><span class="n">list</span><span class="p">())</span>
<span class="o">&gt;&gt;&gt;</span> <span class="p">{</span><span class="n">ObjectID</span><span class="p">(</span><span class="mi">4</span><span class="n">cba8f80c54c6d265b46c2cdfcee6e32348b12be</span><span class="p">):</span> <span class="p">{</span><span class="s1">&#39;construct_duration&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;create_time&#39;</span><span class="p">:</span> <span class="mi">1535223642</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;data_size&#39;</span><span class="p">:</span> <span class="mi">460</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;metadata_size&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;ref_count&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;state&#39;</span><span class="p">:</span> <span class="s1">&#39;sealed&#39;</span><span class="p">},</span>
<span class="o">&gt;&gt;&gt;</span> <span class="n">ObjectID</span><span class="p">(</span><span class="n">a7598230b0c26464c9d9c99ae14773ee81485428</span><span class="p">):</span> <span class="p">{</span><span class="s1">&#39;construct_duration&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;create_time&#39;</span><span class="p">:</span> <span class="mi">1535223644</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;data_size&#39;</span><span class="p">:</span> <span class="mi">460</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;metadata_size&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;ref_count&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;state&#39;</span><span class="p">:</span> <span class="s1">&#39;sealed&#39;</span><span class="p">},</span>
<span class="o">&gt;&gt;&gt;</span> <span class="n">ObjectID</span><span class="p">(</span><span class="n">e603ab0c92098ebf08f90bfcea33ff98f6476870</span><span class="p">):</span> <span class="p">{</span><span class="s1">&#39;construct_duration&#39;</span><span class="p">:</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;create_time&#39;</span><span class="p">:</span> <span class="mi">1535223644</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;data_size&#39;</span><span class="p">:</span> <span class="mi">100</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;metadata_size&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;ref_count&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="o">&gt;&gt;&gt;</span> <span class="s1">&#39;state&#39;</span><span class="p">:</span> <span class="s1">&#39;created&#39;</span><span class="p">}}</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="using-arrow-and-pandas-with-plasma">
<h2>Using Arrow and Pandas with Plasma<a class="headerlink" href="#using-arrow-and-pandas-with-plasma" title="Permalink to this headline"></a></h2>
<div class="section" id="storing-arrow-objects-in-plasma">
<h3>Storing Arrow Objects in Plasma<a class="headerlink" href="#storing-arrow-objects-in-plasma" title="Permalink to this headline"></a></h3>
<p>To store an Arrow object in Plasma, we must first <strong>create</strong> the object and then
<strong>seal</strong> it. However, Arrow objects such as <code class="docutils literal notranslate"><span class="pre">Tensors</span></code> may be more complicated
to write than simple binary data.</p>
<p>To create the object in Plasma, you still need an <code class="docutils literal notranslate"><span class="pre">ObjectID</span></code> and a size to
pass in. To find out the size of your Arrow object, you can use pyarrow
API such as <code class="docutils literal notranslate"><span class="pre">pyarrow.ipc.get_tensor_size</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
<span class="c1"># Create a pyarrow.Tensor object from a numpy random 2-dimensional array</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="n">tensor</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Tensor</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="c1"># Create the object in Plasma</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">bytes</span><span class="p">(</span><span class="mi">20</span><span class="p">))</span>
<span class="n">data_size</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">get_tensor_size</span><span class="p">(</span><span class="n">tensor</span><span class="p">)</span>
<span class="n">buf</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">object_id</span><span class="p">,</span> <span class="n">data_size</span><span class="p">)</span>
</pre></div>
</div>
<p>To write the Arrow <code class="docutils literal notranslate"><span class="pre">Tensor</span></code> object into the buffer, you can use Plasma to
convert the <code class="docutils literal notranslate"><span class="pre">memoryview</span></code> buffer into a <code class="docutils literal notranslate"><span class="pre">pyarrow.FixedSizeBufferWriter</span></code>
object. A <code class="docutils literal notranslate"><span class="pre">pyarrow.FixedSizeBufferWriter</span></code> is a format suitable for Arrow’s
<code class="docutils literal notranslate"><span class="pre">pyarrow.ipc.write_tensor</span></code>:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Write the tensor into the Plasma-allocated buffer</span>
<span class="n">stream</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">FixedSizeBufferWriter</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>
<span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">write_tensor</span><span class="p">(</span><span class="n">tensor</span><span class="p">,</span> <span class="n">stream</span><span class="p">)</span> <span class="c1"># Writes tensor&#39;s 552 bytes to Plasma stream</span>
</pre></div>
</div>
<p>To finish storing the Arrow object in Plasma, call <code class="docutils literal notranslate"><span class="pre">seal</span></code>:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Seal the Plasma object</span>
<span class="n">client</span><span class="o">.</span><span class="n">seal</span><span class="p">(</span><span class="n">object_id</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="getting-arrow-objects-from-plasma">
<h3>Getting Arrow Objects from Plasma<a class="headerlink" href="#getting-arrow-objects-from-plasma" title="Permalink to this headline"></a></h3>
<p>To read the object, first retrieve it as a <code class="docutils literal notranslate"><span class="pre">PlasmaBuffer</span></code> using its object ID.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Get the arrow object by ObjectID.</span>
<span class="p">[</span><span class="n">buf2</span><span class="p">]</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">get_buffers</span><span class="p">([</span><span class="n">object_id</span><span class="p">])</span>
</pre></div>
</div>
<p>To convert the <code class="docutils literal notranslate"><span class="pre">PlasmaBuffer</span></code> back into an Arrow <code class="docutils literal notranslate"><span class="pre">Tensor</span></code>, first create a
pyarrow <code class="docutils literal notranslate"><span class="pre">BufferReader</span></code> object from it. You can then pass the <code class="docutils literal notranslate"><span class="pre">BufferReader</span></code>
into <code class="docutils literal notranslate"><span class="pre">pyarrow.ipc.read_tensor</span></code> to reconstruct the Arrow <code class="docutils literal notranslate"><span class="pre">Tensor</span></code> object:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Reconstruct the Arrow tensor object.</span>
<span class="n">reader</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">BufferReader</span><span class="p">(</span><span class="n">buf2</span><span class="p">)</span>
<span class="n">tensor2</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">read_tensor</span><span class="p">(</span><span class="n">reader</span><span class="p">)</span>
</pre></div>
</div>
<p>Finally, you can use <code class="docutils literal notranslate"><span class="pre">pyarrow.ipc.read_tensor</span></code> to convert the Arrow object
back into numpy data:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Convert back to numpy</span>
<span class="n">array</span> <span class="o">=</span> <span class="n">tensor2</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="storing-pandas-dataframes-in-plasma">
<h3>Storing Pandas DataFrames in Plasma<a class="headerlink" href="#storing-pandas-dataframes-in-plasma" title="Permalink to this headline"></a></h3>
<p>Storing a Pandas <code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> still follows the <strong>create</strong> then <strong>seal</strong>
process of storing an object in the Plasma store, however one cannot directly
write the <code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> to Plasma with Pandas alone. Plasma also needs to know
the size of the <code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> to allocate a buffer for.</p>
<p>See <a class="reference internal" href="pandas.html#pandas-interop"><span class="std std-ref">Pandas Integration</span></a> for more information on using Arrow with Pandas.</p>
<p>You can create the pyarrow equivalent of a Pandas <code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> by using
<code class="docutils literal notranslate"><span class="pre">pyarrow.from_pandas</span></code> to convert it to a <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span>
<span class="c1"># Create a Pandas DataFrame</span>
<span class="n">d</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;one&#39;</span> <span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">([</span><span class="mf">1.</span><span class="p">,</span> <span class="mf">2.</span><span class="p">,</span> <span class="mf">3.</span><span class="p">],</span> <span class="n">index</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;c&#39;</span><span class="p">]),</span>
<span class="s1">&#39;two&#39;</span> <span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">([</span><span class="mf">1.</span><span class="p">,</span> <span class="mf">2.</span><span class="p">,</span> <span class="mf">3.</span><span class="p">,</span> <span class="mf">4.</span><span class="p">],</span> <span class="n">index</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;c&#39;</span><span class="p">,</span> <span class="s1">&#39;d&#39;</span><span class="p">])}</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
<span class="c1"># Convert the Pandas DataFrame into a PyArrow RecordBatch</span>
<span class="n">record_batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_pandas</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</pre></div>
</div>
<p>Creating the Plasma object requires an <code class="docutils literal notranslate"><span class="pre">ObjectID</span></code> and the size of the
data. Now that we have converted the Pandas <code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> into a PyArrow
<code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>, use the <code class="docutils literal notranslate"><span class="pre">MockOutputStream</span></code> to determine the
size of the Plasma object.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Create the Plasma object from the PyArrow RecordBatch. Most of the work here</span>
<span class="c1"># is done to determine the size of buffer to request from the object store.</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">bytes</span><span class="p">(</span><span class="mi">20</span><span class="p">))</span>
<span class="n">mock_sink</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">MockOutputStream</span><span class="p">()</span>
<span class="n">stream_writer</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">mock_sink</span><span class="p">,</span> <span class="n">record_batch</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
<span class="n">stream_writer</span><span class="o">.</span><span class="n">write_batch</span><span class="p">(</span><span class="n">record_batch</span><span class="p">)</span>
<span class="n">stream_writer</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="n">data_size</span> <span class="o">=</span> <span class="n">mock_sink</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="n">buf</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">object_id</span><span class="p">,</span> <span class="n">data_size</span><span class="p">)</span>
</pre></div>
</div>
<p>The DataFrame can now be written to the buffer as follows.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Write the PyArrow RecordBatch to Plasma</span>
<span class="n">stream</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">FixedSizeBufferWriter</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>
<span class="n">stream_writer</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">stream</span><span class="p">,</span> <span class="n">record_batch</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
<span class="n">stream_writer</span><span class="o">.</span><span class="n">write_batch</span><span class="p">(</span><span class="n">record_batch</span><span class="p">)</span>
<span class="n">stream_writer</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
</pre></div>
</div>
<p>Finally, seal the finished object for use by all clients:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Seal the Plasma object</span>
<span class="n">client</span><span class="o">.</span><span class="n">seal</span><span class="p">(</span><span class="n">object_id</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="getting-pandas-dataframes-from-plasma">
<h3>Getting Pandas DataFrames from Plasma<a class="headerlink" href="#getting-pandas-dataframes-from-plasma" title="Permalink to this headline"></a></h3>
<p>Since we store the Pandas DataFrame as a PyArrow <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> object,
to get the object back from the Plasma store, we follow similar steps
to those specified in <a class="reference internal" href="#getting-arrow-objects-from-plasma">Getting Arrow Objects from Plasma</a>.</p>
<p>We first have to convert the <code class="docutils literal notranslate"><span class="pre">PlasmaBuffer</span></code> returned from
<code class="docutils literal notranslate"><span class="pre">client.get_buffers</span></code> into an Arrow <code class="docutils literal notranslate"><span class="pre">BufferReader</span></code> object.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Fetch the Plasma object</span>
<span class="p">[</span><span class="n">data</span><span class="p">]</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">get_buffers</span><span class="p">([</span><span class="n">object_id</span><span class="p">])</span> <span class="c1"># Get PlasmaBuffer from ObjectID</span>
<span class="nb">buffer</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">BufferReader</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
</pre></div>
</div>
<p>From the <code class="docutils literal notranslate"><span class="pre">BufferReader</span></code>, we can create a specific <code class="docutils literal notranslate"><span class="pre">RecordBatchStreamReader</span></code>
in Arrow to reconstruct the stored PyArrow <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> object.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Convert object back into an Arrow RecordBatch</span>
<span class="n">reader</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchStreamReader</span><span class="p">(</span><span class="nb">buffer</span><span class="p">)</span>
<span class="n">record_batch</span> <span class="o">=</span> <span class="n">reader</span><span class="o">.</span><span class="n">read_next_batch</span><span class="p">()</span>
</pre></div>
</div>
<p>The last step is to convert the PyArrow <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> object back into
the original Pandas <code class="docutils literal notranslate"><span class="pre">DataFrame</span></code> structure.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Convert back into Pandas</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">record_batch</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="using-plasma-with-huge-pages">
<h2>Using Plasma with Huge Pages<a class="headerlink" href="#using-plasma-with-huge-pages" title="Permalink to this headline"></a></h2>
<p>On Linux it is possible to use the Plasma store with huge pages for increased
throughput. You first need to create a file system and activate huge pages with</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>sudo mkdir -p /mnt/hugepages
<span class="nv">gid</span><span class="o">=</span><span class="sb">`</span>id -g<span class="sb">`</span>
<span class="nv">uid</span><span class="o">=</span><span class="sb">`</span>id -u<span class="sb">`</span>
sudo mount -t hugetlbfs -o <span class="nv">uid</span><span class="o">=</span><span class="nv">$uid</span> -o <span class="nv">gid</span><span class="o">=</span><span class="nv">$gid</span> none /mnt/hugepages
sudo bash -c <span class="s2">&quot;echo </span><span class="nv">$gid</span><span class="s2"> &gt; /proc/sys/vm/hugetlb_shm_group&quot;</span>
sudo bash -c <span class="s2">&quot;echo 20000 &gt; /proc/sys/vm/nr_hugepages&quot;</span>
</pre></div>
</div>
<p>Note that you only need root access to create the file system, not for
running the object store. You can then start the Plasma store with the <code class="docutils literal notranslate"><span class="pre">-d</span></code>
flag for the mount point of the huge page file system and the <code class="docutils literal notranslate"><span class="pre">-h</span></code> flag
which indicates that huge pages are activated:</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>plasma_store -s /tmp/plasma -m <span class="m">10000000000</span> -d /mnt/hugepages -h
</pre></div>
</div>
<p>You can test this with the following script:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
<span class="kn">import</span> <span class="nn">pyarrow.plasma</span> <span class="kn">as</span> <span class="nn">plasma</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="n">client</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="s2">&quot;/tmp/plasma&quot;</span><span class="p">)</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100000000</span><span class="p">)</span>
<span class="n">tensor</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Tensor</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="n">object_id</span> <span class="o">=</span> <span class="n">plasma</span><span class="o">.</span><span class="n">ObjectID</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">bytes</span><span class="p">(</span><span class="mi">20</span><span class="p">))</span>
<span class="n">buf</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">object_id</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">get_tensor_size</span><span class="p">(</span><span class="n">tensor</span><span class="p">))</span>
<span class="n">stream</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">FixedSizeBufferWriter</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>
<span class="n">stream</span><span class="o">.</span><span class="n">set_memcopy_threads</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
<span class="n">a</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">write_tensor</span><span class="p">(</span><span class="n">tensor</span><span class="p">,</span> <span class="n">stream</span><span class="p">)</span>
<span class="k">print</span><span class="p">(</span><span class="s2">&quot;Writing took &quot;</span><span class="p">,</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">a</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
</div>
<div class='prev-next-bottom'>
<a class='left-prev' id="prev-link" href="generated/pyarrow.HdfsFile.html" title="previous page">pyarrow.HdfsFile</a>
<a class='right-next' id="next-link" href="numpy.html" title="next page">NumPy Integration</a>
</div>
</main>
</div>
</div>
<script src="../_static/js/index.1c5a1a01449ed65a7b51.js"></script>
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
/* We explicitly disable cookie tracking to avoid privacy issues */
_paq.push(['disableCookies']);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '20']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
<footer class="footer mt-5 mt-md-0">
<div class="container">
<div class="footer-item">
<p class="copyright">
&copy; Copyright 2016-2019 Apache Software Foundation.<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 4.0.2.<br/>
</p>
</div>
</div>
</footer>
<script type="text/javascript" src="/docs/_static/versionwarning.js"></script> </body>
</html>