| |
| <!DOCTYPE html> |
| |
| <html> |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" /> |
| |
| <title>Arrow Columnar Format — Apache Arrow v8.0.0</title> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> |
| |
| |
| <link rel="stylesheet" |
| href="../_static/vendor/fontawesome/5.13.0/css/all.min.css"> |
| <link rel="preload" as="font" type="font/woff2" crossorigin |
| href="../_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2"> |
| <link rel="preload" as="font" type="font/woff2" crossorigin |
| href="../_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2"> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/design-style.b7bb847fb20b106c3d81b95245e65545.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"> |
| |
| <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script> |
| <script src="../_static/jquery.js"></script> |
| <script src="../_static/underscore.js"></script> |
| <script src="../_static/doctools.js"></script> |
| <script src="../_static/clipboard.min.js"></script> |
| <script src="../_static/copybutton.js"></script> |
| <script src="../_static/design-tabs.js"></script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/format/Columnar.html" /> |
| <link rel="shortcut icon" href="../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Arrow Flight RPC" href="Flight.html" /> |
| <link rel="prev" title="Format Versioning and Stability" href="Versioning.html" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1" /> |
| <meta name="docsearch:language" content="None"> |
| |
| |
| <!-- Google Analytics --> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| <body data-spy="scroll" data-target="#bd-toc-nav" data-offset="60"> |
| |
| <div class="container-fluid" id="banner"></div> |
| |
| |
| |
| |
| <div class="container-xl"> |
| <div class="row"> |
| |
| |
| <!-- Only show if we have sidebars configured, else just a small margin --> |
| <div class="col-12 col-md-3 bd-sidebar"> |
| <div class="sidebar-start-items"> |
| <a class="navbar-brand" href="../index.html"> |
| <img src="../_static/arrow.png" class="logo" alt="logo"> |
| </a> |
| |
| <div id="version-search-wrapper"> |
| |
| <div id="version-button" class="dropdown"> |
| <button type="button" class="btn btn-secondary btn-sm navbar-btn dropdown-toggle" id="version_switcher_button" data-toggle="dropdown"> |
| 8.0.0 |
| <span class="caret"></span> |
| </button> |
| <div id="version_switcher" class="dropdown-menu list-group-flush py-0" aria-labelledby="version_switcher_button"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| |
| <script type="text/javascript"> |
| // Function to construct the target URL from the JSON components |
| function buildURL(entry) { |
| var template = "https://arrow.apache.org/docs/{version}"; // supplied by jinja |
| template = template.replace("{version}", entry.version); |
| return template; |
| } |
| |
| // Function to check if corresponding page path exists in other version of docs |
| // and, if so, go there instead of the homepage of the other docs version |
| function checkPageExistsAndRedirect(event) { |
| const currentFilePath = "format/Columnar.html", |
| otherDocsHomepage = event.target.getAttribute("href"); |
| let tryUrl = `${otherDocsHomepage}${currentFilePath}`; |
| $.ajax({ |
| type: 'HEAD', |
| url: tryUrl, |
| // if the page exists, go there |
| success: function() { |
| location.href = tryUrl; |
| } |
| }).fail(function() { |
| location.href = otherDocsHomepage; |
| }); |
| return false; |
| } |
| |
| // Function to populate the version switcher |
| (function () { |
| // get JSON config |
| $.getJSON("/docs/_static/versions.json", function(data, textStatus, jqXHR) { |
| // create the nodes first (before AJAX calls) to ensure the order is |
| // correct (for now, links will go to doc version homepage) |
| $.each(data, function(index, entry) { |
| // if no custom name specified (e.g., "latest"), use version string |
| if (!("name" in entry)) { |
| entry.name = entry.version; |
| } |
| // construct the appropriate URL, and add it to the dropdown |
| entry.url = buildURL(entry); |
| const node = document.createElement("a"); |
| node.setAttribute("class", "list-group-item list-group-item-action py-1"); |
| node.setAttribute("href", `${entry.url}`); |
| node.textContent = `${entry.name}`; |
| node.onclick = checkPageExistsAndRedirect; |
| $("#version_switcher").append(node); |
| }); |
| }); |
| })(); |
| </script> |
| |
| <form id="search-box" class="bd-search d-flex align-items-center" action="../search.html" method="get"> |
| <i class="icon fas fa-search"></i> |
| <input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" > |
| </form> |
| |
| </div> |
| |
| <nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation"> |
| <div class="bd-toc-item active"> |
| |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Supported Environments |
| </span> |
| </p> |
| <ul class="nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/> |
| <label for="toctree-checkbox-1"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../cpp/getting_started.html"> |
| User Guide |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/> |
| <label for="toctree-checkbox-2"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/overview.html"> |
| High-Level Overview |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/conventions.html"> |
| Conventions |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/build_system.html"> |
| Using Arrow C++ in your own project |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/memory.html"> |
| Memory Management |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/arrays.html"> |
| Arrays |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/datatypes.html"> |
| Data Types |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/tables.html"> |
| Tabular Data |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/compute.html"> |
| Compute Functions |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/streaming_execution.html"> |
| Streaming execution engine |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/io.html"> |
| Input / output and filesystems |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/ipc.html"> |
| Reading and writing the Arrow IPC format |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/orc.html"> |
| Reading and Writing ORC files |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/parquet.html"> |
| Reading and writing Parquet files |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/csv.html"> |
| Reading and Writing CSV files |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/json.html"> |
| Reading JSON files |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/dataset.html"> |
| Tabular Datasets |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/gdb.html"> |
| Debugging code using Arrow |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/threading.html"> |
| Thread Management |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/env_vars.html"> |
| Environment Variables |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../cpp/examples/index.html"> |
| Examples |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/> |
| <label for="toctree-checkbox-3"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/examples/cmake_minimal_build.html"> |
| Minimal build using CMake |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/examples/compute_and_write_example.html"> |
| Compute and Write CSV Example |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/examples/dataset_documentation_example.html"> |
| Arrow Datasets example |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/examples/row_columnar_conversion.html"> |
| Row to columnar conversion |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/examples/tuple_range_conversion.html"> |
| std::tuple-like ranges to Arrow |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../cpp/api.html"> |
| API Reference |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/> |
| <label for="toctree-checkbox-4"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/support.html"> |
| Programming Support |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/memory.html"> |
| Memory (management) |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/thread.html"> |
| Thread (management) |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/datatype.html"> |
| Data Types |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/array.html"> |
| Arrays |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/scalar.html"> |
| Scalars |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/builder.html"> |
| Array Builders |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/table.html"> |
| Two-dimensional Datasets |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/c_abi.html"> |
| C Interfaces |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/compute.html"> |
| Compute Functions |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/tensor.html"> |
| Tensors |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/utilities.html"> |
| Utilities |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/async.html"> |
| Asynchronous programming |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/io.html"> |
| Input / output |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/ipc.html"> |
| Arrow IPC |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/formats.html"> |
| File Formats |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/cuda.html"> |
| CUDA support |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/flightsql.html"> |
| Arrow Flight SQL |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/filesystem.html"> |
| Filesystems |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../cpp/api/dataset.html"> |
| Dataset |
| </a> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://pkg.go.dev/github.com/apache/arrow/go"> |
| Go |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../java/index.html"> |
| Java |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/> |
| <label for="toctree-checkbox-5"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/quickstartguide.html"> |
| Quick Start Guide |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/overview.html"> |
| High-Level Overview |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/install.html"> |
| Installing Java Modules |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/memory.html"> |
| Memory Management |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/vector.html"> |
| ValueVector |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/vector_schema_root.html"> |
| Tabular Data |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/ipc.html"> |
| Reading/Writing IPC formats |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/algorithm.html"> |
| Java Algorithms |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/dataset.html"> |
| Dataset |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/cdata.html"> |
| C Data Interface |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../java/reference/index.html"> |
| Reference (javadoc) |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow-julia/blob/main/README.md"> |
| Julia |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../python/index.html"> |
| Python |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/> |
| <label for="toctree-checkbox-6"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/install.html"> |
| Installing PyArrow |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/getstarted.html"> |
| Getting Started |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/data.html"> |
| Data Types and In-Memory Data Model |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/compute.html"> |
| Compute Functions |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/memory.html"> |
| Memory and IO Interfaces |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/ipc.html"> |
| Streaming, Serialization, and IPC |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/filesystems.html"> |
| Filesystem Interface |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../python/filesystems_deprecated.html"> |
| Filesystem Interface (legacy) |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/> |
| <label for="toctree-checkbox-7"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.hdfs.connect.html"> |
| pyarrow.hdfs.connect |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.cat.html"> |
| pyarrow.HadoopFileSystem.cat |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.chmod.html"> |
| pyarrow.HadoopFileSystem.chmod |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.chown.html"> |
| pyarrow.HadoopFileSystem.chown |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.delete.html"> |
| pyarrow.HadoopFileSystem.delete |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.df.html"> |
| pyarrow.HadoopFileSystem.df |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.disk_usage.html"> |
| pyarrow.HadoopFileSystem.disk_usage |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.download.html"> |
| pyarrow.HadoopFileSystem.download |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.exists.html"> |
| pyarrow.HadoopFileSystem.exists |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.get_capacity.html"> |
| pyarrow.HadoopFileSystem.get_capacity |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.get_space_used.html"> |
| pyarrow.HadoopFileSystem.get_space_used |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.info.html"> |
| pyarrow.HadoopFileSystem.info |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.ls.html"> |
| pyarrow.HadoopFileSystem.ls |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.mkdir.html"> |
| pyarrow.HadoopFileSystem.mkdir |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.open.html"> |
| pyarrow.HadoopFileSystem.open |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.rename.html"> |
| pyarrow.HadoopFileSystem.rename |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.rm.html"> |
| pyarrow.HadoopFileSystem.rm |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HadoopFileSystem.upload.html"> |
| pyarrow.HadoopFileSystem.upload |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/generated/pyarrow.HdfsFile.html"> |
| pyarrow.HdfsFile |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/plasma.html"> |
| The Plasma In-Memory Object Store |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/numpy.html"> |
| NumPy Integration |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/pandas.html"> |
| Pandas Integration |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/timestamps.html"> |
| Timestamps |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/orc.html"> |
| Reading and Writing the Apache ORC Format |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/csv.html"> |
| Reading and Writing CSV files |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/feather.html"> |
| Feather File Format |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/json.html"> |
| Reading JSON files |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/parquet.html"> |
| Reading and Writing the Apache Parquet Format |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/dataset.html"> |
| Tabular Datasets |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/extending_types.html"> |
| Extending pyarrow |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../python/integration.html"> |
| PyArrow Integrations |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/> |
| <label for="toctree-checkbox-8"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/integration/python_r.html"> |
| Integrating PyArrow with R |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/integration/python_java.html"> |
| Integrating PyArrow with Java |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/integration/extending.html"> |
| Using pyarrow from C++ and Cython Code |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../python/integration/cuda.html"> |
| CUDA Integration |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/env_vars.html"> |
| Environment Variables |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../python/api.html"> |
| API Reference |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/> |
| <label for="toctree-checkbox-9"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/datatypes.html"> |
| Data Types and Schemas |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/> |
| <label for="toctree-checkbox-10"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.null.html"> |
| pyarrow.null |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.bool_.html"> |
| pyarrow.bool_ |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.int8.html"> |
| pyarrow.int8 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.int16.html"> |
| pyarrow.int16 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.int32.html"> |
| pyarrow.int32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.int64.html"> |
| pyarrow.int64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.uint8.html"> |
| pyarrow.uint8 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.uint16.html"> |
| pyarrow.uint16 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.uint32.html"> |
| pyarrow.uint32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.uint64.html"> |
| pyarrow.uint64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.float16.html"> |
| pyarrow.float16 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.float32.html"> |
| pyarrow.float32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.float64.html"> |
| pyarrow.float64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.time32.html"> |
| pyarrow.time32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.time64.html"> |
| pyarrow.time64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.timestamp.html"> |
| pyarrow.timestamp |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.date32.html"> |
| pyarrow.date32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.date64.html"> |
| pyarrow.date64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.duration.html"> |
| pyarrow.duration |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.month_day_nano_interval.html"> |
| pyarrow.month_day_nano_interval |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.binary.html"> |
| pyarrow.binary |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.string.html"> |
| pyarrow.string |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.utf8.html"> |
| pyarrow.utf8 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.large_binary.html"> |
| pyarrow.large_binary |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.large_string.html"> |
| pyarrow.large_string |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.large_utf8.html"> |
| pyarrow.large_utf8 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.decimal128.html"> |
| pyarrow.decimal128 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.list_.html"> |
| pyarrow.list_ |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.large_list.html"> |
| pyarrow.large_list |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.map_.html"> |
| pyarrow.map_ |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.struct.html"> |
| pyarrow.struct |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dictionary.html"> |
| pyarrow.dictionary |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.field.html"> |
| pyarrow.field |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.schema.html"> |
| pyarrow.schema |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.from_numpy_dtype.html"> |
| pyarrow.from_numpy_dtype |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.unify_schemas.html"> |
| pyarrow.unify_schemas |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.DataType.html"> |
| pyarrow.DataType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.DictionaryType.html"> |
| pyarrow.DictionaryType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ListType.html"> |
| pyarrow.ListType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.MapType.html"> |
| pyarrow.MapType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.StructType.html"> |
| pyarrow.StructType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UnionType.html"> |
| pyarrow.UnionType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.TimestampType.html"> |
| pyarrow.TimestampType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Time32Type.html"> |
| pyarrow.Time32Type |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Time64Type.html"> |
| pyarrow.Time64Type |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.FixedSizeBinaryType.html"> |
| pyarrow.FixedSizeBinaryType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Decimal128Type.html"> |
| pyarrow.Decimal128Type |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Field.html"> |
| pyarrow.Field |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Schema.html"> |
| pyarrow.Schema |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ExtensionType.html"> |
| pyarrow.ExtensionType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.PyExtensionType.html"> |
| pyarrow.PyExtensionType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.register_extension_type.html"> |
| pyarrow.register_extension_type |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.unregister_extension_type.html"> |
| pyarrow.unregister_extension_type |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_boolean.html"> |
| pyarrow.types.is_boolean |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_integer.html"> |
| pyarrow.types.is_integer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_signed_integer.html"> |
| pyarrow.types.is_signed_integer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_unsigned_integer.html"> |
| pyarrow.types.is_unsigned_integer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_int8.html"> |
| pyarrow.types.is_int8 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_int16.html"> |
| pyarrow.types.is_int16 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_int32.html"> |
| pyarrow.types.is_int32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_int64.html"> |
| pyarrow.types.is_int64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_uint8.html"> |
| pyarrow.types.is_uint8 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_uint16.html"> |
| pyarrow.types.is_uint16 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_uint32.html"> |
| pyarrow.types.is_uint32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_uint64.html"> |
| pyarrow.types.is_uint64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_floating.html"> |
| pyarrow.types.is_floating |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_float16.html"> |
| pyarrow.types.is_float16 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_float32.html"> |
| pyarrow.types.is_float32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_float64.html"> |
| pyarrow.types.is_float64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_decimal.html"> |
| pyarrow.types.is_decimal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_list.html"> |
| pyarrow.types.is_list |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_large_list.html"> |
| pyarrow.types.is_large_list |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_struct.html"> |
| pyarrow.types.is_struct |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_union.html"> |
| pyarrow.types.is_union |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_nested.html"> |
| pyarrow.types.is_nested |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_temporal.html"> |
| pyarrow.types.is_temporal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_timestamp.html"> |
| pyarrow.types.is_timestamp |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_date.html"> |
| pyarrow.types.is_date |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_date32.html"> |
| pyarrow.types.is_date32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_date64.html"> |
| pyarrow.types.is_date64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_time.html"> |
| pyarrow.types.is_time |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_time32.html"> |
| pyarrow.types.is_time32 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_time64.html"> |
| pyarrow.types.is_time64 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_null.html"> |
| pyarrow.types.is_null |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_binary.html"> |
| pyarrow.types.is_binary |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_unicode.html"> |
| pyarrow.types.is_unicode |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_string.html"> |
| pyarrow.types.is_string |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_large_binary.html"> |
| pyarrow.types.is_large_binary |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_large_unicode.html"> |
| pyarrow.types.is_large_unicode |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_large_string.html"> |
| pyarrow.types.is_large_string |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_fixed_size_binary.html"> |
| pyarrow.types.is_fixed_size_binary |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_map.html"> |
| pyarrow.types.is_map |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.types.is_dictionary.html"> |
| pyarrow.types.is_dictionary |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/arrays.html"> |
| Arrays and Scalars |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" type="checkbox"/> |
| <label for="toctree-checkbox-11"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.array.html"> |
| pyarrow.array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.nulls.html"> |
| pyarrow.nulls |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Array.html"> |
| pyarrow.Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.BooleanArray.html"> |
| pyarrow.BooleanArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.FloatingPointArray.html"> |
| pyarrow.FloatingPointArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.IntegerArray.html"> |
| pyarrow.IntegerArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int8Array.html"> |
| pyarrow.Int8Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int16Array.html"> |
| pyarrow.Int16Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int32Array.html"> |
| pyarrow.Int32Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int64Array.html"> |
| pyarrow.Int64Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.NullArray.html"> |
| pyarrow.NullArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.NumericArray.html"> |
| pyarrow.NumericArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt8Array.html"> |
| pyarrow.UInt8Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt16Array.html"> |
| pyarrow.UInt16Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt32Array.html"> |
| pyarrow.UInt32Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt64Array.html"> |
| pyarrow.UInt64Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.BinaryArray.html"> |
| pyarrow.BinaryArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.StringArray.html"> |
| pyarrow.StringArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.FixedSizeBinaryArray.html"> |
| pyarrow.FixedSizeBinaryArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.LargeBinaryArray.html"> |
| pyarrow.LargeBinaryArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.LargeStringArray.html"> |
| pyarrow.LargeStringArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Time32Array.html"> |
| pyarrow.Time32Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Time64Array.html"> |
| pyarrow.Time64Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Date32Array.html"> |
| pyarrow.Date32Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Date64Array.html"> |
| pyarrow.Date64Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.TimestampArray.html"> |
| pyarrow.TimestampArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.DurationArray.html"> |
| pyarrow.DurationArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.MonthDayNanoIntervalArray.html"> |
| pyarrow.MonthDayNanoIntervalArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Decimal128Array.html"> |
| pyarrow.Decimal128Array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.DictionaryArray.html"> |
| pyarrow.DictionaryArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ListArray.html"> |
| pyarrow.ListArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.FixedSizeListArray.html"> |
| pyarrow.FixedSizeListArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.LargeListArray.html"> |
| pyarrow.LargeListArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.MapArray.html"> |
| pyarrow.MapArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.StructArray.html"> |
| pyarrow.StructArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UnionArray.html"> |
| pyarrow.UnionArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ExtensionArray.html"> |
| pyarrow.ExtensionArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.scalar.html"> |
| pyarrow.scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.NA.html"> |
| pyarrow.NA |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Scalar.html"> |
| pyarrow.Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.BooleanScalar.html"> |
| pyarrow.BooleanScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int8Scalar.html"> |
| pyarrow.Int8Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int16Scalar.html"> |
| pyarrow.Int16Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int32Scalar.html"> |
| pyarrow.Int32Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Int64Scalar.html"> |
| pyarrow.Int64Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt8Scalar.html"> |
| pyarrow.UInt8Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt16Scalar.html"> |
| pyarrow.UInt16Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt32Scalar.html"> |
| pyarrow.UInt32Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UInt64Scalar.html"> |
| pyarrow.UInt64Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.FloatScalar.html"> |
| pyarrow.FloatScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.DoubleScalar.html"> |
| pyarrow.DoubleScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.BinaryScalar.html"> |
| pyarrow.BinaryScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.StringScalar.html"> |
| pyarrow.StringScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.FixedSizeBinaryScalar.html"> |
| pyarrow.FixedSizeBinaryScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.LargeBinaryScalar.html"> |
| pyarrow.LargeBinaryScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.LargeStringScalar.html"> |
| pyarrow.LargeStringScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Time32Scalar.html"> |
| pyarrow.Time32Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Time64Scalar.html"> |
| pyarrow.Time64Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Date32Scalar.html"> |
| pyarrow.Date32Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Date64Scalar.html"> |
| pyarrow.Date64Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.TimestampScalar.html"> |
| pyarrow.TimestampScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.DurationScalar.html"> |
| pyarrow.DurationScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.MonthDayNanoIntervalScalar.html"> |
| pyarrow.MonthDayNanoIntervalScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Decimal128Scalar.html"> |
| pyarrow.Decimal128Scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.DictionaryScalar.html"> |
| pyarrow.DictionaryScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ListScalar.html"> |
| pyarrow.ListScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.LargeListScalar.html"> |
| pyarrow.LargeListScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.MapScalar.html"> |
| pyarrow.MapScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.StructScalar.html"> |
| pyarrow.StructScalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.UnionScalar.html"> |
| pyarrow.UnionScalar |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/memory.html"> |
| Buffers and Memory |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" type="checkbox"/> |
| <label for="toctree-checkbox-12"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.allocate_buffer.html"> |
| pyarrow.allocate_buffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.py_buffer.html"> |
| pyarrow.py_buffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.foreign_buffer.html"> |
| pyarrow.foreign_buffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Buffer.html"> |
| pyarrow.Buffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ResizableBuffer.html"> |
| pyarrow.ResizableBuffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Codec.html"> |
| pyarrow.Codec |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compress.html"> |
| pyarrow.compress |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.decompress.html"> |
| pyarrow.decompress |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.MemoryPool.html"> |
| pyarrow.MemoryPool |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.default_memory_pool.html"> |
| pyarrow.default_memory_pool |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.jemalloc_memory_pool.html"> |
| pyarrow.jemalloc_memory_pool |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.mimalloc_memory_pool.html"> |
| pyarrow.mimalloc_memory_pool |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.system_memory_pool.html"> |
| pyarrow.system_memory_pool |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.jemalloc_set_decay_ms.html"> |
| pyarrow.jemalloc_set_decay_ms |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.set_memory_pool.html"> |
| pyarrow.set_memory_pool |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.log_memory_allocations.html"> |
| pyarrow.log_memory_allocations |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.total_allocated_bytes.html"> |
| pyarrow.total_allocated_bytes |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/compute.html"> |
| Compute Functions |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-13" name="toctree-checkbox-13" type="checkbox"/> |
| <label for="toctree-checkbox-13"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.all.html"> |
| pyarrow.compute.all |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.any.html"> |
| pyarrow.compute.any |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.approximate_median.html"> |
| pyarrow.compute.approximate_median |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.count.html"> |
| pyarrow.compute.count |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.count_distinct.html"> |
| pyarrow.compute.count_distinct |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.index.html"> |
| pyarrow.compute.index |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.max.html"> |
| pyarrow.compute.max |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.mean.html"> |
| pyarrow.compute.mean |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.min.html"> |
| pyarrow.compute.min |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.min_max.html"> |
| pyarrow.compute.min_max |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.mode.html"> |
| pyarrow.compute.mode |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.product.html"> |
| pyarrow.compute.product |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.quantile.html"> |
| pyarrow.compute.quantile |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.stddev.html"> |
| pyarrow.compute.stddev |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.sum.html"> |
| pyarrow.compute.sum |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.tdigest.html"> |
| pyarrow.compute.tdigest |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.variance.html"> |
| pyarrow.compute.variance |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.abs.html"> |
| pyarrow.compute.abs |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.abs_checked.html"> |
| pyarrow.compute.abs_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.add.html"> |
| pyarrow.compute.add |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.add_checked.html"> |
| pyarrow.compute.add_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.divide.html"> |
| pyarrow.compute.divide |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.divide_checked.html"> |
| pyarrow.compute.divide_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.multiply.html"> |
| pyarrow.compute.multiply |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.multiply_checked.html"> |
| pyarrow.compute.multiply_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.negate.html"> |
| pyarrow.compute.negate |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.negate_checked.html"> |
| pyarrow.compute.negate_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.power.html"> |
| pyarrow.compute.power |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.power_checked.html"> |
| pyarrow.compute.power_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.sign.html"> |
| pyarrow.compute.sign |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.sqrt.html"> |
| pyarrow.compute.sqrt |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.sqrt_checked.html"> |
| pyarrow.compute.sqrt_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.subtract.html"> |
| pyarrow.compute.subtract |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.subtract_checked.html"> |
| pyarrow.compute.subtract_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.bit_wise_and.html"> |
| pyarrow.compute.bit_wise_and |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.bit_wise_not.html"> |
| pyarrow.compute.bit_wise_not |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.bit_wise_or.html"> |
| pyarrow.compute.bit_wise_or |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.bit_wise_xor.html"> |
| pyarrow.compute.bit_wise_xor |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.shift_left.html"> |
| pyarrow.compute.shift_left |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.shift_left_checked.html"> |
| pyarrow.compute.shift_left_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.shift_right.html"> |
| pyarrow.compute.shift_right |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.shift_right_checked.html"> |
| pyarrow.compute.shift_right_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ceil.html"> |
| pyarrow.compute.ceil |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.floor.html"> |
| pyarrow.compute.floor |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.round.html"> |
| pyarrow.compute.round |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.round_to_multiple.html"> |
| pyarrow.compute.round_to_multiple |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.trunc.html"> |
| pyarrow.compute.trunc |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ln.html"> |
| pyarrow.compute.ln |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ln_checked.html"> |
| pyarrow.compute.ln_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.log10.html"> |
| pyarrow.compute.log10 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.log10_checked.html"> |
| pyarrow.compute.log10_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.log1p.html"> |
| pyarrow.compute.log1p |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.log1p_checked.html"> |
| pyarrow.compute.log1p_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.log2.html"> |
| pyarrow.compute.log2 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.log2_checked.html"> |
| pyarrow.compute.log2_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.logb.html"> |
| pyarrow.compute.logb |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.logb_checked.html"> |
| pyarrow.compute.logb_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.acos.html"> |
| pyarrow.compute.acos |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.acos_checked.html"> |
| pyarrow.compute.acos_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.asin.html"> |
| pyarrow.compute.asin |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.asin_checked.html"> |
| pyarrow.compute.asin_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.atan.html"> |
| pyarrow.compute.atan |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.atan2.html"> |
| pyarrow.compute.atan2 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.cos.html"> |
| pyarrow.compute.cos |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.cos_checked.html"> |
| pyarrow.compute.cos_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.sin.html"> |
| pyarrow.compute.sin |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.sin_checked.html"> |
| pyarrow.compute.sin_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.tan.html"> |
| pyarrow.compute.tan |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.tan_checked.html"> |
| pyarrow.compute.tan_checked |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.equal.html"> |
| pyarrow.compute.equal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.greater.html"> |
| pyarrow.compute.greater |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.greater_equal.html"> |
| pyarrow.compute.greater_equal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.less.html"> |
| pyarrow.compute.less |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.less_equal.html"> |
| pyarrow.compute.less_equal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.not_equal.html"> |
| pyarrow.compute.not_equal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.max_element_wise.html"> |
| pyarrow.compute.max_element_wise |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.min_element_wise.html"> |
| pyarrow.compute.min_element_wise |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.and_.html"> |
| pyarrow.compute.and_ |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.and_kleene.html"> |
| pyarrow.compute.and_kleene |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.and_not.html"> |
| pyarrow.compute.and_not |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.and_not_kleene.html"> |
| pyarrow.compute.and_not_kleene |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.invert.html"> |
| pyarrow.compute.invert |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.or_.html"> |
| pyarrow.compute.or_ |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.or_kleene.html"> |
| pyarrow.compute.or_kleene |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.xor.html"> |
| pyarrow.compute.xor |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_alnum.html"> |
| pyarrow.compute.ascii_is_alnum |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_alpha.html"> |
| pyarrow.compute.ascii_is_alpha |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_decimal.html"> |
| pyarrow.compute.ascii_is_decimal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_lower.html"> |
| pyarrow.compute.ascii_is_lower |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_printable.html"> |
| pyarrow.compute.ascii_is_printable |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_space.html"> |
| pyarrow.compute.ascii_is_space |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_upper.html"> |
| pyarrow.compute.ascii_is_upper |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_alnum.html"> |
| pyarrow.compute.utf8_is_alnum |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_alpha.html"> |
| pyarrow.compute.utf8_is_alpha |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_decimal.html"> |
| pyarrow.compute.utf8_is_decimal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_digit.html"> |
| pyarrow.compute.utf8_is_digit |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_lower.html"> |
| pyarrow.compute.utf8_is_lower |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_numeric.html"> |
| pyarrow.compute.utf8_is_numeric |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_printable.html"> |
| pyarrow.compute.utf8_is_printable |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_space.html"> |
| pyarrow.compute.utf8_is_space |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_upper.html"> |
| pyarrow.compute.utf8_is_upper |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_is_title.html"> |
| pyarrow.compute.ascii_is_title |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_is_title.html"> |
| pyarrow.compute.utf8_is_title |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.string_is_ascii.html"> |
| pyarrow.compute.string_is_ascii |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_capitalize.html"> |
| pyarrow.compute.ascii_capitalize |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_lower.html"> |
| pyarrow.compute.ascii_lower |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_reverse.html"> |
| pyarrow.compute.ascii_reverse |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_swapcase.html"> |
| pyarrow.compute.ascii_swapcase |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_title.html"> |
| pyarrow.compute.ascii_title |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_upper.html"> |
| pyarrow.compute.ascii_upper |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.binary_length.html"> |
| pyarrow.compute.binary_length |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.binary_repeat.html"> |
| pyarrow.compute.binary_repeat |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.binary_replace_slice.html"> |
| pyarrow.compute.binary_replace_slice |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.binary_reverse.html"> |
| pyarrow.compute.binary_reverse |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.replace_substring.html"> |
| pyarrow.compute.replace_substring |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.replace_substring_regex.html"> |
| pyarrow.compute.replace_substring_regex |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_capitalize.html"> |
| pyarrow.compute.utf8_capitalize |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_length.html"> |
| pyarrow.compute.utf8_length |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_lower.html"> |
| pyarrow.compute.utf8_lower |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_replace_slice.html"> |
| pyarrow.compute.utf8_replace_slice |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_reverse.html"> |
| pyarrow.compute.utf8_reverse |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_swapcase.html"> |
| pyarrow.compute.utf8_swapcase |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_title.html"> |
| pyarrow.compute.utf8_title |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_upper.html"> |
| pyarrow.compute.utf8_upper |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_center.html"> |
| pyarrow.compute.ascii_center |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_lpad.html"> |
| pyarrow.compute.ascii_lpad |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_rpad.html"> |
| pyarrow.compute.ascii_rpad |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_center.html"> |
| pyarrow.compute.utf8_center |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_lpad.html"> |
| pyarrow.compute.utf8_lpad |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_rpad.html"> |
| pyarrow.compute.utf8_rpad |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_ltrim.html"> |
| pyarrow.compute.ascii_ltrim |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_ltrim_whitespace.html"> |
| pyarrow.compute.ascii_ltrim_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_rtrim.html"> |
| pyarrow.compute.ascii_rtrim |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_rtrim_whitespace.html"> |
| pyarrow.compute.ascii_rtrim_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_trim.html"> |
| pyarrow.compute.ascii_trim |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_trim_whitespace.html"> |
| pyarrow.compute.ascii_trim_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_ltrim.html"> |
| pyarrow.compute.utf8_ltrim |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_ltrim_whitespace.html"> |
| pyarrow.compute.utf8_ltrim_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_rtrim.html"> |
| pyarrow.compute.utf8_rtrim |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_rtrim_whitespace.html"> |
| pyarrow.compute.utf8_rtrim_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_trim.html"> |
| pyarrow.compute.utf8_trim |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_trim_whitespace.html"> |
| pyarrow.compute.utf8_trim_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ascii_split_whitespace.html"> |
| pyarrow.compute.ascii_split_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.split_pattern.html"> |
| pyarrow.compute.split_pattern |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.split_pattern_regex.html"> |
| pyarrow.compute.split_pattern_regex |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_split_whitespace.html"> |
| pyarrow.compute.utf8_split_whitespace |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.extract_regex.html"> |
| pyarrow.compute.extract_regex |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.binary_join.html"> |
| pyarrow.compute.binary_join |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.binary_join_element_wise.html"> |
| pyarrow.compute.binary_join_element_wise |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.utf8_slice_codeunits.html"> |
| pyarrow.compute.utf8_slice_codeunits |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.count_substring.html"> |
| pyarrow.compute.count_substring |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.count_substring_regex.html"> |
| pyarrow.compute.count_substring_regex |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ends_with.html"> |
| pyarrow.compute.ends_with |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.find_substring.html"> |
| pyarrow.compute.find_substring |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.find_substring_regex.html"> |
| pyarrow.compute.find_substring_regex |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.index_in.html"> |
| pyarrow.compute.index_in |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.is_in.html"> |
| pyarrow.compute.is_in |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.match_like.html"> |
| pyarrow.compute.match_like |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.match_substring.html"> |
| pyarrow.compute.match_substring |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.match_substring_regex.html"> |
| pyarrow.compute.match_substring_regex |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.starts_with.html"> |
| pyarrow.compute.starts_with |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.indices_nonzero.html"> |
| pyarrow.compute.indices_nonzero |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.is_finite.html"> |
| pyarrow.compute.is_finite |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.is_inf.html"> |
| pyarrow.compute.is_inf |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.is_nan.html"> |
| pyarrow.compute.is_nan |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.is_null.html"> |
| pyarrow.compute.is_null |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.is_valid.html"> |
| pyarrow.compute.is_valid |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.true_unless_null.html"> |
| pyarrow.compute.true_unless_null |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.case_when.html"> |
| pyarrow.compute.case_when |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.choose.html"> |
| pyarrow.compute.choose |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.coalesce.html"> |
| pyarrow.compute.coalesce |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.if_else.html"> |
| pyarrow.compute.if_else |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.cast.html"> |
| pyarrow.compute.cast |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ceil_temporal.html"> |
| pyarrow.compute.ceil_temporal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.floor_temporal.html"> |
| pyarrow.compute.floor_temporal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.round_temporal.html"> |
| pyarrow.compute.round_temporal |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.strftime.html"> |
| pyarrow.compute.strftime |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.strptime.html"> |
| pyarrow.compute.strptime |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.day.html"> |
| pyarrow.compute.day |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.day_of_week.html"> |
| pyarrow.compute.day_of_week |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.day_of_year.html"> |
| pyarrow.compute.day_of_year |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.hour.html"> |
| pyarrow.compute.hour |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.iso_week.html"> |
| pyarrow.compute.iso_week |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.iso_year.html"> |
| pyarrow.compute.iso_year |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.iso_calendar.html"> |
| pyarrow.compute.iso_calendar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.is_leap_year.html"> |
| pyarrow.compute.is_leap_year |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.microsecond.html"> |
| pyarrow.compute.microsecond |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.millisecond.html"> |
| pyarrow.compute.millisecond |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.minute.html"> |
| pyarrow.compute.minute |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.month.html"> |
| pyarrow.compute.month |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.nanosecond.html"> |
| pyarrow.compute.nanosecond |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.quarter.html"> |
| pyarrow.compute.quarter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.second.html"> |
| pyarrow.compute.second |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.subsecond.html"> |
| pyarrow.compute.subsecond |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.us_week.html"> |
| pyarrow.compute.us_week |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.us_year.html"> |
| pyarrow.compute.us_year |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.week.html"> |
| pyarrow.compute.week |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.year.html"> |
| pyarrow.compute.year |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.year_month_day.html"> |
| pyarrow.compute.year_month_day |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.day_time_interval_between.html"> |
| pyarrow.compute.day_time_interval_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.days_between.html"> |
| pyarrow.compute.days_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.hours_between.html"> |
| pyarrow.compute.hours_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.microseconds_between.html"> |
| pyarrow.compute.microseconds_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.milliseconds_between.html"> |
| pyarrow.compute.milliseconds_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.minutes_between.html"> |
| pyarrow.compute.minutes_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.month_day_nano_interval_between.html"> |
| pyarrow.compute.month_day_nano_interval_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.month_interval_between.html"> |
| pyarrow.compute.month_interval_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.nanoseconds_between.html"> |
| pyarrow.compute.nanoseconds_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.quarters_between.html"> |
| pyarrow.compute.quarters_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.seconds_between.html"> |
| pyarrow.compute.seconds_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.weeks_between.html"> |
| pyarrow.compute.weeks_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.years_between.html"> |
| pyarrow.compute.years_between |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.assume_timezone.html"> |
| pyarrow.compute.assume_timezone |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.dictionary_encode.html"> |
| pyarrow.compute.dictionary_encode |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.unique.html"> |
| pyarrow.compute.unique |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.value_counts.html"> |
| pyarrow.compute.value_counts |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.array_filter.html"> |
| pyarrow.compute.array_filter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.array_take.html"> |
| pyarrow.compute.array_take |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.drop_null.html"> |
| pyarrow.compute.drop_null |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.filter.html"> |
| pyarrow.compute.filter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.take.html"> |
| pyarrow.compute.take |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.array_sort_indices.html"> |
| pyarrow.compute.array_sort_indices |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.partition_nth_indices.html"> |
| pyarrow.compute.partition_nth_indices |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.select_k_unstable.html"> |
| pyarrow.compute.select_k_unstable |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.sort_indices.html"> |
| pyarrow.compute.sort_indices |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.fill_null_backward.html"> |
| pyarrow.compute.fill_null_backward |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.fill_null_forward.html"> |
| pyarrow.compute.fill_null_forward |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.list_element.html"> |
| pyarrow.compute.list_element |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.list_flatten.html"> |
| pyarrow.compute.list_flatten |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.list_parent_indices.html"> |
| pyarrow.compute.list_parent_indices |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.list_value_length.html"> |
| pyarrow.compute.list_value_length |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.make_struct.html"> |
| pyarrow.compute.make_struct |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.map_lookup.html"> |
| pyarrow.compute.map_lookup |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.replace_with_mask.html"> |
| pyarrow.compute.replace_with_mask |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.struct_field.html"> |
| pyarrow.compute.struct_field |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ArraySortOptions.html"> |
| pyarrow.compute.ArraySortOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.AssumeTimezoneOptions.html"> |
| pyarrow.compute.AssumeTimezoneOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.CastOptions.html"> |
| pyarrow.compute.CastOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.CountOptions.html"> |
| pyarrow.compute.CountOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.CountOptions.html"> |
| pyarrow.compute.CountOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.DayOfWeekOptions.html"> |
| pyarrow.compute.DayOfWeekOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.DictionaryEncodeOptions.html"> |
| pyarrow.compute.DictionaryEncodeOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ElementWiseAggregateOptions.html"> |
| pyarrow.compute.ElementWiseAggregateOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ExtractRegexOptions.html"> |
| pyarrow.compute.ExtractRegexOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.FilterOptions.html"> |
| pyarrow.compute.FilterOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.IndexOptions.html"> |
| pyarrow.compute.IndexOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.JoinOptions.html"> |
| pyarrow.compute.JoinOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.MakeStructOptions.html"> |
| pyarrow.compute.MakeStructOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.MapLookupOptions.html"> |
| pyarrow.compute.MapLookupOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.MatchSubstringOptions.html"> |
| pyarrow.compute.MatchSubstringOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ModeOptions.html"> |
| pyarrow.compute.ModeOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.NullOptions.html"> |
| pyarrow.compute.NullOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.PadOptions.html"> |
| pyarrow.compute.PadOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.PartitionNthOptions.html"> |
| pyarrow.compute.PartitionNthOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.QuantileOptions.html"> |
| pyarrow.compute.QuantileOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ReplaceSliceOptions.html"> |
| pyarrow.compute.ReplaceSliceOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ReplaceSubstringOptions.html"> |
| pyarrow.compute.ReplaceSubstringOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.RoundOptions.html"> |
| pyarrow.compute.RoundOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.RoundTemporalOptions.html"> |
| pyarrow.compute.RoundTemporalOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.RoundToMultipleOptions.html"> |
| pyarrow.compute.RoundToMultipleOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ScalarAggregateOptions.html"> |
| pyarrow.compute.ScalarAggregateOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.ScalarAggregateOptions.html"> |
| pyarrow.compute.ScalarAggregateOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.SelectKOptions.html"> |
| pyarrow.compute.SelectKOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.SetLookupOptions.html"> |
| pyarrow.compute.SetLookupOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.SliceOptions.html"> |
| pyarrow.compute.SliceOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.SortOptions.html"> |
| pyarrow.compute.SortOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.SplitOptions.html"> |
| pyarrow.compute.SplitOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.SplitPatternOptions.html"> |
| pyarrow.compute.SplitPatternOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.StrftimeOptions.html"> |
| pyarrow.compute.StrftimeOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.StrptimeOptions.html"> |
| pyarrow.compute.StrptimeOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.StructFieldOptions.html"> |
| pyarrow.compute.StructFieldOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.TakeOptions.html"> |
| pyarrow.compute.TakeOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.TDigestOptions.html"> |
| pyarrow.compute.TDigestOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.TDigestOptions.html"> |
| pyarrow.compute.TDigestOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.TrimOptions.html"> |
| pyarrow.compute.TrimOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.VarianceOptions.html"> |
| pyarrow.compute.VarianceOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.compute.WeekOptions.html"> |
| pyarrow.compute.WeekOptions |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/files.html"> |
| Streams and File Access |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-14" name="toctree-checkbox-14" type="checkbox"/> |
| <label for="toctree-checkbox-14"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.input_stream.html"> |
| pyarrow.input_stream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.output_stream.html"> |
| pyarrow.output_stream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.memory_map.html"> |
| pyarrow.memory_map |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.create_memory_map.html"> |
| pyarrow.create_memory_map |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.NativeFile.html"> |
| pyarrow.NativeFile |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.OSFile.html"> |
| pyarrow.OSFile |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.PythonFile.html"> |
| pyarrow.PythonFile |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.BufferReader.html"> |
| pyarrow.BufferReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.BufferOutputStream.html"> |
| pyarrow.BufferOutputStream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.FixedSizeBufferWriter.html"> |
| pyarrow.FixedSizeBufferWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.MemoryMappedFile.html"> |
| pyarrow.MemoryMappedFile |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.CompressedInputStream.html"> |
| pyarrow.CompressedInputStream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.CompressedOutputStream.html"> |
| pyarrow.CompressedOutputStream |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/tables.html"> |
| Tables and Tensors |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-15" name="toctree-checkbox-15" type="checkbox"/> |
| <label for="toctree-checkbox-15"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.chunked_array.html"> |
| pyarrow.chunked_array |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.concat_arrays.html"> |
| pyarrow.concat_arrays |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.concat_tables.html"> |
| pyarrow.concat_tables |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.record_batch.html"> |
| pyarrow.record_batch |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.table.html"> |
| pyarrow.table |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ChunkedArray.html"> |
| pyarrow.ChunkedArray |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.RecordBatch.html"> |
| pyarrow.RecordBatch |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Table.html"> |
| pyarrow.Table |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.TableGroupBy.html"> |
| pyarrow.TableGroupBy |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.RecordBatchReader.html"> |
| pyarrow.RecordBatchReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.Tensor.html"> |
| pyarrow.Tensor |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/ipc.html"> |
| Serialization and IPC |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-16" name="toctree-checkbox-16" type="checkbox"/> |
| <label for="toctree-checkbox-16"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.new_file.html"> |
| pyarrow.ipc.new_file |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.open_file.html"> |
| pyarrow.ipc.open_file |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.new_stream.html"> |
| pyarrow.ipc.new_stream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.open_stream.html"> |
| pyarrow.ipc.open_stream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.read_message.html"> |
| pyarrow.ipc.read_message |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.read_record_batch.html"> |
| pyarrow.ipc.read_record_batch |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.get_record_batch_size.html"> |
| pyarrow.ipc.get_record_batch_size |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.read_tensor.html"> |
| pyarrow.ipc.read_tensor |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.write_tensor.html"> |
| pyarrow.ipc.write_tensor |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.get_tensor_size.html"> |
| pyarrow.ipc.get_tensor_size |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.IpcReadOptions.html"> |
| pyarrow.ipc.IpcReadOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.IpcWriteOptions.html"> |
| pyarrow.ipc.IpcWriteOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.Message.html"> |
| pyarrow.ipc.Message |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.MessageReader.html"> |
| pyarrow.ipc.MessageReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.RecordBatchFileReader.html"> |
| pyarrow.ipc.RecordBatchFileReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.RecordBatchFileWriter.html"> |
| pyarrow.ipc.RecordBatchFileWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.RecordBatchStreamReader.html"> |
| pyarrow.ipc.RecordBatchStreamReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.ipc.RecordBatchStreamWriter.html"> |
| pyarrow.ipc.RecordBatchStreamWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.serialize.html"> |
| pyarrow.serialize |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.serialize_to.html"> |
| pyarrow.serialize_to |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.deserialize.html"> |
| pyarrow.deserialize |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.deserialize_components.html"> |
| pyarrow.deserialize_components |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.deserialize_from.html"> |
| pyarrow.deserialize_from |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.read_serialized.html"> |
| pyarrow.read_serialized |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.SerializedPyObject.html"> |
| pyarrow.SerializedPyObject |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.SerializationContext.html"> |
| pyarrow.SerializationContext |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/flight.html"> |
| Arrow Flight |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-17" name="toctree-checkbox-17" type="checkbox"/> |
| <label for="toctree-checkbox-17"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.Action.html"> |
| pyarrow.flight.Action |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ActionType.html"> |
| pyarrow.flight.ActionType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.DescriptorType.html"> |
| pyarrow.flight.DescriptorType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightDescriptor.html"> |
| pyarrow.flight.FlightDescriptor |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightEndpoint.html"> |
| pyarrow.flight.FlightEndpoint |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightInfo.html"> |
| pyarrow.flight.FlightInfo |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.Location.html"> |
| pyarrow.flight.Location |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.MetadataRecordBatchReader.html"> |
| pyarrow.flight.MetadataRecordBatchReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.MetadataRecordBatchWriter.html"> |
| pyarrow.flight.MetadataRecordBatchWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.Ticket.html"> |
| pyarrow.flight.Ticket |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.Result.html"> |
| pyarrow.flight.Result |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.connect.html"> |
| pyarrow.flight.connect |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightCallOptions.html"> |
| pyarrow.flight.FlightCallOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightClient.html"> |
| pyarrow.flight.FlightClient |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightStreamReader.html"> |
| pyarrow.flight.FlightStreamReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightStreamWriter.html"> |
| pyarrow.flight.FlightStreamWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ClientMiddlewareFactory.html"> |
| pyarrow.flight.ClientMiddlewareFactory |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ClientMiddleware.html"> |
| pyarrow.flight.ClientMiddleware |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightDataStream.html"> |
| pyarrow.flight.FlightDataStream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightMetadataWriter.html"> |
| pyarrow.flight.FlightMetadataWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightServerBase.html"> |
| pyarrow.flight.FlightServerBase |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.GeneratorStream.html"> |
| pyarrow.flight.GeneratorStream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.RecordBatchStream.html"> |
| pyarrow.flight.RecordBatchStream |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ServerCallContext.html"> |
| pyarrow.flight.ServerCallContext |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ServerMiddlewareFactory.html"> |
| pyarrow.flight.ServerMiddlewareFactory |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ServerMiddleware.html"> |
| pyarrow.flight.ServerMiddleware |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ClientAuthHandler.html"> |
| pyarrow.flight.ClientAuthHandler |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.ServerAuthHandler.html"> |
| pyarrow.flight.ServerAuthHandler |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightError.html"> |
| pyarrow.flight.FlightError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightCancelledError.html"> |
| pyarrow.flight.FlightCancelledError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightInternalError.html"> |
| pyarrow.flight.FlightInternalError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightServerError.html"> |
| pyarrow.flight.FlightServerError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightTimedOutError.html"> |
| pyarrow.flight.FlightTimedOutError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightUnauthenticatedError.html"> |
| pyarrow.flight.FlightUnauthenticatedError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightUnauthorizedError.html"> |
| pyarrow.flight.FlightUnauthorizedError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightUnavailableError.html"> |
| pyarrow.flight.FlightUnavailableError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightWriteSizeExceededError.html"> |
| pyarrow.flight.FlightWriteSizeExceededError |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.FlightMethod.html"> |
| pyarrow.flight.FlightMethod |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.flight.CallInfo.html"> |
| pyarrow.flight.CallInfo |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/formats.html"> |
| Tabular File Formats |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-18" name="toctree-checkbox-18" type="checkbox"/> |
| <label for="toctree-checkbox-18"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.ConvertOptions.html"> |
| pyarrow.csv.ConvertOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.CSVStreamingReader.html"> |
| pyarrow.csv.CSVStreamingReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.CSVWriter.html"> |
| pyarrow.csv.CSVWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.ISO8601.html"> |
| pyarrow.csv.ISO8601 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.ParseOptions.html"> |
| pyarrow.csv.ParseOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.ReadOptions.html"> |
| pyarrow.csv.ReadOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.WriteOptions.html"> |
| pyarrow.csv.WriteOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.open_csv.html"> |
| pyarrow.csv.open_csv |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.read_csv.html"> |
| pyarrow.csv.read_csv |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.write_csv.html"> |
| pyarrow.csv.write_csv |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.csv.InvalidRow.html"> |
| pyarrow.csv.InvalidRow |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.feather.read_feather.html"> |
| pyarrow.feather.read_feather |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.feather.read_table.html"> |
| pyarrow.feather.read_table |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.feather.write_feather.html"> |
| pyarrow.feather.write_feather |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.json.ReadOptions.html"> |
| pyarrow.json.ReadOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.json.ParseOptions.html"> |
| pyarrow.json.ParseOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.json.read_json.html"> |
| pyarrow.json.read_json |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.ParquetDataset.html"> |
| pyarrow.parquet.ParquetDataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.ParquetFile.html"> |
| pyarrow.parquet.ParquetFile |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.ParquetWriter.html"> |
| pyarrow.parquet.ParquetWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.read_table.html"> |
| pyarrow.parquet.read_table |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.read_metadata.html"> |
| pyarrow.parquet.read_metadata |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.read_pandas.html"> |
| pyarrow.parquet.read_pandas |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.read_schema.html"> |
| pyarrow.parquet.read_schema |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.write_metadata.html"> |
| pyarrow.parquet.write_metadata |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.write_table.html"> |
| pyarrow.parquet.write_table |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.write_to_dataset.html"> |
| pyarrow.parquet.write_to_dataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.FileMetaData.html"> |
| pyarrow.parquet.FileMetaData |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.RowGroupMetaData.html"> |
| pyarrow.parquet.RowGroupMetaData |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.ColumnChunkMetaData.html"> |
| pyarrow.parquet.ColumnChunkMetaData |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.Statistics.html"> |
| pyarrow.parquet.Statistics |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.ParquetSchema.html"> |
| pyarrow.parquet.ParquetSchema |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.ColumnSchema.html"> |
| pyarrow.parquet.ColumnSchema |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.ParquetLogicalType.html"> |
| pyarrow.parquet.ParquetLogicalType |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.encryption.CryptoFactory.html"> |
| pyarrow.parquet.encryption.CryptoFactory |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.encryption.KmsClient.html"> |
| pyarrow.parquet.encryption.KmsClient |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.encryption.KmsConnectionConfig.html"> |
| pyarrow.parquet.encryption.KmsConnectionConfig |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.encryption.EncryptionConfiguration.html"> |
| pyarrow.parquet.encryption.EncryptionConfiguration |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.parquet.encryption.DecryptionConfiguration.html"> |
| pyarrow.parquet.encryption.DecryptionConfiguration |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.orc.ORCFile.html"> |
| pyarrow.orc.ORCFile |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.orc.ORCWriter.html"> |
| pyarrow.orc.ORCWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.orc.read_table.html"> |
| pyarrow.orc.read_table |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.orc.write_table.html"> |
| pyarrow.orc.write_table |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/filesystems.html"> |
| Filesystems |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-19" name="toctree-checkbox-19" type="checkbox"/> |
| <label for="toctree-checkbox-19"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.FileInfo.html"> |
| pyarrow.fs.FileInfo |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.FileSelector.html"> |
| pyarrow.fs.FileSelector |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.FileSystem.html"> |
| pyarrow.fs.FileSystem |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.LocalFileSystem.html"> |
| pyarrow.fs.LocalFileSystem |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.S3FileSystem.html"> |
| pyarrow.fs.S3FileSystem |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.HadoopFileSystem.html"> |
| pyarrow.fs.HadoopFileSystem |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.SubTreeFileSystem.html"> |
| pyarrow.fs.SubTreeFileSystem |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.PyFileSystem.html"> |
| pyarrow.fs.PyFileSystem |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.FileSystemHandler.html"> |
| pyarrow.fs.FileSystemHandler |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.FSSpecHandler.html"> |
| pyarrow.fs.FSSpecHandler |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.copy_files.html"> |
| pyarrow.fs.copy_files |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.initialize_s3.html"> |
| pyarrow.fs.initialize_s3 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.finalize_s3.html"> |
| pyarrow.fs.finalize_s3 |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.resolve_s3_region.html"> |
| pyarrow.fs.resolve_s3_region |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.fs.S3LogLevel.html"> |
| pyarrow.fs.S3LogLevel |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/dataset.html"> |
| Dataset |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-20" name="toctree-checkbox-20" type="checkbox"/> |
| <label for="toctree-checkbox-20"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.dataset.html"> |
| pyarrow.dataset.dataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.parquet_dataset.html"> |
| pyarrow.dataset.parquet_dataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.partitioning.html"> |
| pyarrow.dataset.partitioning |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.field.html"> |
| pyarrow.dataset.field |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.scalar.html"> |
| pyarrow.dataset.scalar |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.write_dataset.html"> |
| pyarrow.dataset.write_dataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.FileFormat.html"> |
| pyarrow.dataset.FileFormat |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.CsvFileFormat.html"> |
| pyarrow.dataset.CsvFileFormat |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.CsvFragmentScanOptions.html"> |
| pyarrow.dataset.CsvFragmentScanOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.IpcFileFormat.html"> |
| pyarrow.dataset.IpcFileFormat |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.ParquetFileFormat.html"> |
| pyarrow.dataset.ParquetFileFormat |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.ParquetReadOptions.html"> |
| pyarrow.dataset.ParquetReadOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.ParquetFragmentScanOptions.html"> |
| pyarrow.dataset.ParquetFragmentScanOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.OrcFileFormat.html"> |
| pyarrow.dataset.OrcFileFormat |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.Partitioning.html"> |
| pyarrow.dataset.Partitioning |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.PartitioningFactory.html"> |
| pyarrow.dataset.PartitioningFactory |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.DirectoryPartitioning.html"> |
| pyarrow.dataset.DirectoryPartitioning |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.HivePartitioning.html"> |
| pyarrow.dataset.HivePartitioning |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.FilenamePartitioning.html"> |
| pyarrow.dataset.FilenamePartitioning |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.Dataset.html"> |
| pyarrow.dataset.Dataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.FileSystemDataset.html"> |
| pyarrow.dataset.FileSystemDataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.FileSystemFactoryOptions.html"> |
| pyarrow.dataset.FileSystemFactoryOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.FileSystemDatasetFactory.html"> |
| pyarrow.dataset.FileSystemDatasetFactory |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.UnionDataset.html"> |
| pyarrow.dataset.UnionDataset |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.Fragment.html"> |
| pyarrow.dataset.Fragment |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.FragmentScanOptions.html"> |
| pyarrow.dataset.FragmentScanOptions |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.TaggedRecordBatch.html"> |
| pyarrow.dataset.TaggedRecordBatch |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.Scanner.html"> |
| pyarrow.dataset.Scanner |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.Expression.html"> |
| pyarrow.dataset.Expression |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.dataset.InMemoryDataset.html"> |
| pyarrow.dataset.InMemoryDataset |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/plasma.html"> |
| Plasma In-Memory Object Store |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-21" name="toctree-checkbox-21" type="checkbox"/> |
| <label for="toctree-checkbox-21"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.plasma.ObjectID.html"> |
| pyarrow.plasma.ObjectID |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.plasma.PlasmaClient.html"> |
| pyarrow.plasma.PlasmaClient |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.plasma.PlasmaBuffer.html"> |
| pyarrow.plasma.PlasmaBuffer |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/cuda.html"> |
| CUDA Integration |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-22" name="toctree-checkbox-22" type="checkbox"/> |
| <label for="toctree-checkbox-22"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.Context.html"> |
| pyarrow.cuda.Context |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.CudaBuffer.html"> |
| pyarrow.cuda.CudaBuffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.new_host_buffer.html"> |
| pyarrow.cuda.new_host_buffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.HostBuffer.html"> |
| pyarrow.cuda.HostBuffer |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.BufferReader.html"> |
| pyarrow.cuda.BufferReader |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.BufferWriter.html"> |
| pyarrow.cuda.BufferWriter |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.serialize_record_batch.html"> |
| pyarrow.cuda.serialize_record_batch |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.read_record_batch.html"> |
| pyarrow.cuda.read_record_batch |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.read_message.html"> |
| pyarrow.cuda.read_message |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cuda.IpcMemHandle.html"> |
| pyarrow.cuda.IpcMemHandle |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l3 has-children"> |
| <a class="reference internal" href="../python/api/misc.html"> |
| Miscellaneous |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-23" name="toctree-checkbox-23" type="checkbox"/> |
| <label for="toctree-checkbox-23"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.cpu_count.html"> |
| pyarrow.cpu_count |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.set_cpu_count.html"> |
| pyarrow.set_cpu_count |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.io_thread_count.html"> |
| pyarrow.io_thread_count |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.set_io_thread_count.html"> |
| pyarrow.set_io_thread_count |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.get_include.html"> |
| pyarrow.get_include |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.get_libraries.html"> |
| pyarrow.get_libraries |
| </a> |
| </li> |
| <li class="toctree-l4"> |
| <a class="reference internal" href="../python/generated/pyarrow.get_library_dirs.html"> |
| pyarrow.get_library_dirs |
| </a> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/getting_involved.html"> |
| Getting Involved |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../python/benchmarks.html"> |
| Benchmarks |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| </ul> |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Cookbooks |
| </span> |
| </p> |
| <ul class="nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://arrow.apache.org/cookbook/java/"> |
| Java |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://arrow.apache.org/cookbook/py/"> |
| Python |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://arrow.apache.org/cookbook/r/"> |
| R |
| </a> |
| </li> |
| </ul> |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Specifications and Protocols |
| </span> |
| </p> |
| <ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="Versioning.html"> |
| Format Versioning and Stability |
| </a> |
| </li> |
| <li class="toctree-l1 current active"> |
| <a class="current reference internal" href="#"> |
| Arrow Columnar Format |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="Flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="FlightSql.html"> |
| Arrow Flight SQL |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="Integration.html"> |
| Integration Testing |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="CDataInterface.html"> |
| The Arrow C data interface |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="CStreamInterface.html"> |
| The Arrow C stream interface |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="Other.html"> |
| Other Data Structures |
| </a> |
| </li> |
| </ul> |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Development |
| </span> |
| </p> |
| <ul class="nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/contributing.html"> |
| Contributing to Apache Arrow |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/bug_reports.html"> |
| Report bugs and propose features |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../developers/guide/index.html"> |
| New Contributor’s Guide |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-24" name="toctree-checkbox-24" type="checkbox"/> |
| <label for="toctree-checkbox-24"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/guide/architectural_overview.html"> |
| Architectural Overview |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/guide/communication.html"> |
| Communication |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../developers/guide/step_by_step/index.html"> |
| Steps in making your first PR |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-25" name="toctree-checkbox-25" type="checkbox"/> |
| <label for="toctree-checkbox-25"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/step_by_step/set_up.html"> |
| Set up |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/step_by_step/building.html"> |
| Building the Arrow libraries 🏋🏿♀️ |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/step_by_step/finding_issues.html"> |
| Finding good first issues 🔎 |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/step_by_step/arrow_codebase.html"> |
| Working on the Arrow codebase 🧐 |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/step_by_step/testing.html"> |
| Testing 🧪 |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/step_by_step/styling.html"> |
| Styling 😎 |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/step_by_step/pr_lifecycle.html"> |
| Lifecycle of a pull request |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/guide/documentation.html"> |
| Helping with documentation |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../developers/guide/tutorials/index.html"> |
| Tutorials |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-26" name="toctree-checkbox-26" type="checkbox"/> |
| <label for="toctree-checkbox-26"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/tutorials/python_tutorial.html"> |
| Python tutorial |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../developers/guide/tutorials/r_tutorial.html"> |
| R tutorials |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/guide/resources.html"> |
| Additional information and resources |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/overview.html"> |
| Contributing Overview |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/reviewing.html"> |
| Reviewing contributions |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../developers/cpp/index.html"> |
| C++ Development |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-27" name="toctree-checkbox-27" type="checkbox"/> |
| <label for="toctree-checkbox-27"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/cpp/building.html"> |
| Building Arrow C++ |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/cpp/development.html"> |
| Development Guidelines |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/cpp/windows.html"> |
| Developing on Windows |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/cpp/conventions.html"> |
| Conventions |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/cpp/fuzzing.html"> |
| Fuzzing Arrow C++ |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../developers/java/index.html"> |
| Java Development |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-28" name="toctree-checkbox-28" type="checkbox"/> |
| <label for="toctree-checkbox-28"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/java/building.html"> |
| Building Arrow Java |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/java/development.html"> |
| Development Guidelines |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/python.html"> |
| Python Development |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../developers/continuous_integration/index.html"> |
| Continuous Integration |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-29" name="toctree-checkbox-29" type="checkbox"/> |
| <label for="toctree-checkbox-29"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/continuous_integration/overview.html"> |
| Continuous Integration |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/continuous_integration/docker.html"> |
| Running Docker Builds |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/continuous_integration/archery.html"> |
| Daily Development using Archery |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../developers/continuous_integration/crossbow.html"> |
| Packaging and Testing with Crossbow |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/benchmarks.html"> |
| Benchmarks |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/documentation.html"> |
| Building the Documentation |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../developers/computeir.html"> |
| Arrow Compute IR (Intermediate Representation) |
| </a> |
| </li> |
| </ul> |
| |
| |
| </div> |
| </nav> |
| </div> |
| <div class="sidebar-end-items"> |
| </div> |
| </div> |
| |
| |
| |
| |
| <div class="d-none d-xl-block col-xl-2 bd-toc"> |
| |
| |
| <div class="toc-item"> |
| |
| <div class="tocsection onthispage mt-5 pt-1 pb-3"> |
| <i class="fas fa-list"></i> On this page |
| </div> |
| |
| <nav id="bd-toc-nav"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#terminology"> |
| Terminology |
| </a> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#physical-memory-layout"> |
| Physical Memory Layout |
| </a> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#buffer-alignment-and-padding"> |
| Buffer Alignment and Padding |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#array-lengths"> |
| Array lengths |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#null-count"> |
| Null count |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#validity-bitmaps"> |
| Validity bitmaps |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#fixed-size-primitive-layout"> |
| Fixed-size Primitive Layout |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#variable-size-binary-layout"> |
| Variable-size Binary Layout |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#variable-size-list-layout"> |
| Variable-size List Layout |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#fixed-size-list-layout"> |
| Fixed-Size List Layout |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#struct-layout"> |
| Struct Layout |
| </a> |
| <ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#struct-validity"> |
| Struct Validity |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#union-layout"> |
| Union Layout |
| </a> |
| <ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#dense-union"> |
| Dense Union |
| </a> |
| </li> |
| <li class="toc-h4 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#sparse-union"> |
| Sparse Union |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#null-layout"> |
| Null Layout |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#dictionary-encoded-layout"> |
| Dictionary-encoded Layout |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#buffer-listing-for-each-layout"> |
| Buffer Listing for Each Layout |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#logical-types"> |
| Logical Types |
| </a> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#serialization-and-interprocess-communication-ipc"> |
| Serialization and Interprocess Communication (IPC) |
| </a> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#encapsulated-message-format"> |
| Encapsulated message format |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#schema-message"> |
| Schema message |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#recordbatch-message"> |
| RecordBatch message |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#byte-order-endianness"> |
| Byte Order (Endianness) |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#ipc-streaming-format"> |
| IPC Streaming Format |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#ipc-file-format"> |
| IPC File Format |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#dictionary-messages"> |
| Dictionary Messages |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#custom-application-metadata"> |
| Custom Application Metadata |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#extension-types"> |
| Extension Types |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#implementation-guidelines"> |
| Implementation guidelines |
| </a> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#implementing-a-subset-the-spec"> |
| Implementing a subset the spec |
| </a> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"> |
| <a class="reference internal nav-link" href="#extensibility"> |
| Extensibility |
| </a> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| |
| </nav> |
| </div> |
| |
| <div class="toc-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/master/docs/source/format/Columnar.rst"> |
| <i class="fas fa-pencil-alt"></i> Edit this page |
| </a> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| <main class="col-12 col-md-9 col-xl-7 py-md-5 pl-md-5 pr-md-4 bd-content" role="main"> |
| |
| <div> |
| |
| <section id="arrow-columnar-format"> |
| <span id="format-columnar"></span><h1>Arrow Columnar Format<a class="headerlink" href="#arrow-columnar-format" title="Permalink to this headline">#</a></h1> |
| <p><em>Version: 1.0</em></p> |
| <p>The “Arrow Columnar Format” includes a language-agnostic in-memory |
| data structure specification, metadata serialization, and a protocol |
| for serialization and generic data transport.</p> |
| <p>This document is intended to provide adequate detail to create a new |
| implementation of the columnar format without the aid of an existing |
| implementation. We utilize Google’s <a class="reference external" href="http://github.com/google/flatbuffers">Flatbuffers</a> project for |
| metadata serialization, so it will be necessary to refer to the |
| project’s <a class="reference external" href="https://github.com/apache/arrow/tree/master/format">Flatbuffers protocol definition files</a> |
| while reading this document.</p> |
| <p>The columnar format has some key features:</p> |
| <ul class="simple"> |
| <li><p>Data adjacency for sequential access (scans)</p></li> |
| <li><p>O(1) (constant-time) random access</p></li> |
| <li><p>SIMD and vectorization-friendly</p></li> |
| <li><p>Relocatable without “pointer swizzling”, allowing for true zero-copy |
| access in shared memory</p></li> |
| </ul> |
| <p>The Arrow columnar format provides analytical performance and data |
| locality guarantees in exchange for comparatively more expensive |
| mutation operations. This document is concerned only with in-memory |
| data representation and serialization details; issues such as |
| coordinating mutation of data structures are left to be handled by |
| implementations.</p> |
| <section id="terminology"> |
| <h2>Terminology<a class="headerlink" href="#terminology" title="Permalink to this headline">#</a></h2> |
| <p>Since different projects have used different words to describe various |
| concepts, here is a small glossary to help disambiguate.</p> |
| <ul class="simple"> |
| <li><p><strong>Array</strong> or <strong>Vector</strong>: a sequence of values with known length all |
| having the same type. These terms are used interchangeably in |
| different Arrow implementations, but we use “array” in this |
| document.</p></li> |
| <li><p><strong>Slot</strong>: a single logical value in an array of some particular data type</p></li> |
| <li><p><strong>Buffer</strong> or <strong>Contiguous memory region</strong>: a sequential virtual |
| address space with a given length. Any byte can be reached via a |
| single pointer offset less than the region’s length.</p></li> |
| <li><p><strong>Physical Layout</strong>: The underlying memory layout for an array |
| without taking into account any value semantics. For example, a |
| 32-bit signed integer array and 32-bit floating point array have the |
| same layout.</p></li> |
| <li><p><strong>Parent</strong> and <strong>child arrays</strong>: names to express relationships |
| between physical value arrays in a nested type structure. For |
| example, a <code class="docutils literal notranslate"><span class="pre">List<T></span></code>-type parent array has a T-type array as its |
| child (see more on lists below).</p></li> |
| <li><p><strong>Primitive type</strong>: a data type having no child types. This includes |
| such types as fixed bit-width, variable-size binary, and null types.</p></li> |
| <li><p><strong>Nested type</strong>: a data type whose full structure depends on one or |
| more other child types. Two fully-specified nested types are equal |
| if and only if their child types are equal. For example, <code class="docutils literal notranslate"><span class="pre">List<U></span></code> |
| is distinct from <code class="docutils literal notranslate"><span class="pre">List<V></span></code> iff U and V are different types.</p></li> |
| <li><p><strong>Logical type</strong>: An application-facing semantic value type that is |
| implemented using some physical layout. For example, Decimal |
| values are stored as 16 bytes in a fixed-size binary |
| layout. Similarly, strings can be stored as <code class="docutils literal notranslate"><span class="pre">List<1-byte></span></code>. A |
| timestamp may be stored as 64-bit fixed-size layout.</p></li> |
| </ul> |
| </section> |
| <section id="physical-memory-layout"> |
| <span id="format-layout"></span><h2>Physical Memory Layout<a class="headerlink" href="#physical-memory-layout" title="Permalink to this headline">#</a></h2> |
| <p>Arrays are defined by a few pieces of metadata and data:</p> |
| <ul class="simple"> |
| <li><p>A logical data type.</p></li> |
| <li><p>A sequence of buffers.</p></li> |
| <li><p>A length as a 64-bit signed integer. Implementations are permitted |
| to be limited to 32-bit lengths, see more on this below.</p></li> |
| <li><p>A null count as a 64-bit signed integer.</p></li> |
| <li><p>An optional <strong>dictionary</strong>, for dictionary-encoded arrays.</p></li> |
| </ul> |
| <p>Nested arrays additionally have a sequence of one or more sets of |
| these items, called the <strong>child arrays</strong>.</p> |
| <p>Each logical data type has a well-defined physical layout. Here are |
| the different physical layouts defined by Arrow:</p> |
| <ul class="simple"> |
| <li><p><strong>Primitive (fixed-size)</strong>: a sequence of values each having the |
| same byte or bit width</p></li> |
| <li><p><strong>Variable-size Binary</strong>: a sequence of values each having a variable |
| byte length. Two variants of this layout are supported using 32-bit |
| and 64-bit length encoding.</p></li> |
| <li><p><strong>Fixed-size List</strong>: a nested layout where each value has the same |
| number of elements taken from a child data type.</p></li> |
| <li><p><strong>Variable-size List</strong>: a nested layout where each value is a |
| variable-length sequence of values taken from a child data type. Two |
| variants of this layout are supported using 32-bit and 64-bit length |
| encoding.</p></li> |
| <li><p><strong>Struct</strong>: a nested layout consisting of a collection of named |
| child <strong>fields</strong> each having the same length but possibly different |
| types.</p></li> |
| <li><p><strong>Sparse</strong> and <strong>Dense Union</strong>: a nested layout representing a |
| sequence of values, each of which can have type chosen from a |
| collection of child array types.</p></li> |
| <li><p><strong>Null</strong>: a sequence of all null values, having null logical type</p></li> |
| </ul> |
| <p>The Arrow columnar memory layout only applies to <em>data</em> and not |
| <em>metadata</em>. Implementations are free to represent metadata in-memory |
| in whichever form is convenient for them. We handle metadata |
| <strong>serialization</strong> in an implementation-independent way using |
| <a class="reference external" href="http://github.com/google/flatbuffers">Flatbuffers</a>, detailed below.</p> |
| <section id="buffer-alignment-and-padding"> |
| <h3>Buffer Alignment and Padding<a class="headerlink" href="#buffer-alignment-and-padding" title="Permalink to this headline">#</a></h3> |
| <p>Implementations are recommended to allocate memory on aligned |
| addresses (multiple of 8- or 64-bytes) and pad (overallocate) to a |
| length that is a multiple of 8 or 64 bytes. When serializing Arrow |
| data for interprocess communication, these alignment and padding |
| requirements are enforced. If possible, we suggest that you prefer |
| using 64-byte alignment and padding. Unless otherwise noted, padded |
| bytes do not need to have a specific value.</p> |
| <p>The alignment requirement follows best practices for optimized memory |
| access:</p> |
| <ul class="simple"> |
| <li><p>Elements in numeric arrays will be guaranteed to be retrieved via aligned access.</p></li> |
| <li><p>On some architectures alignment can help limit partially used cache lines.</p></li> |
| </ul> |
| <p>The recommendation for 64 byte alignment comes from the <a class="reference external" href="https://software.intel.com/en-us/articles/practical-intel-avx-optimization-on-2nd-generation-intel-core-processors">Intel |
| performance guide</a> that recommends alignment of memory to match SIMD |
| register width. The specific padding length was chosen because it |
| matches the largest SIMD instruction registers available on widely |
| deployed x86 architecture (Intel AVX-512).</p> |
| <p>The recommended padding of 64 bytes allows for using <a class="reference external" href="https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-introduction-to-the-simd-data-layout-templates">SIMD</a> |
| instructions consistently in loops without additional conditional |
| checks. This should allow for simpler, efficient and CPU |
| cache-friendly code. In other words, we can load the entire 64-byte |
| buffer into a 512-bit wide SIMD register and get data-level |
| parallelism on all the columnar values packed into the 64-byte |
| buffer. Guaranteed padding can also allow certain compilers to |
| generate more optimized code directly (e.g. One can safely use Intel’s |
| <code class="docutils literal notranslate"><span class="pre">-qopt-assume-safe-padding</span></code>).</p> |
| </section> |
| <section id="array-lengths"> |
| <h3>Array lengths<a class="headerlink" href="#array-lengths" title="Permalink to this headline">#</a></h3> |
| <p>Array lengths are represented in the Arrow metadata as a 64-bit signed |
| integer. An implementation of Arrow is considered valid even if it only |
| supports lengths up to the maximum 32-bit signed integer, though. If using |
| Arrow in a multi-language environment, we recommend limiting lengths to |
| 2 <sup>31</sup> - 1 elements or less. Larger data sets can be represented using |
| multiple array chunks.</p> |
| </section> |
| <section id="null-count"> |
| <h3>Null count<a class="headerlink" href="#null-count" title="Permalink to this headline">#</a></h3> |
| <p>The number of null value slots is a property of the physical array and |
| considered part of the data structure. The null count is represented |
| in the Arrow metadata as a 64-bit signed integer, as it may be as |
| large as the array length.</p> |
| </section> |
| <section id="validity-bitmaps"> |
| <h3>Validity bitmaps<a class="headerlink" href="#validity-bitmaps" title="Permalink to this headline">#</a></h3> |
| <p>Any value in an array may be semantically null, whether primitive or nested |
| type.</p> |
| <p>All array types, with the exception of union types (more on these later), |
| utilize a dedicated memory buffer, known as the validity (or “null”) bitmap, to |
| encode the nullness or non-nullness of each value slot. The validity bitmap |
| must be large enough to have at least 1 bit for each array slot.</p> |
| <p>Whether any array slot is valid (non-null) is encoded in the respective bits of |
| this bitmap. A 1 (set bit) for index <code class="docutils literal notranslate"><span class="pre">j</span></code> indicates that the value is not null, |
| while a 0 (bit not set) indicates that it is null. Bitmaps are to be |
| initialized to be all unset at allocation time (this includes padding):</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">is_valid</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> <span class="o">-></span> <span class="n">bitmap</span><span class="p">[</span><span class="n">j</span> <span class="o">/</span> <span class="mi">8</span><span class="p">]</span> <span class="o">&</span> <span class="p">(</span><span class="mi">1</span> <span class="o"><<</span> <span class="p">(</span><span class="n">j</span> <span class="o">%</span> <span class="mi">8</span><span class="p">))</span> |
| </pre></div> |
| </div> |
| <p>We use <a class="reference external" href="https://en.wikipedia.org/wiki/Bit_numbering">least-significant bit (LSB) numbering</a> (also known as |
| bit-endianness). This means that within a group of 8 bits, we read |
| right-to-left:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> |
| |
| <span class="n">bitmap</span> |
| <span class="n">j</span> <span class="n">mod</span> <span class="mi">8</span> <span class="mi">7</span> <span class="mi">6</span> <span class="mi">5</span> <span class="mi">4</span> <span class="mi">3</span> <span class="mi">2</span> <span class="mi">1</span> <span class="mi">0</span> |
| <span class="mi">0</span> <span class="mi">0</span> <span class="mi">1</span> <span class="mi">0</span> <span class="mi">1</span> <span class="mi">0</span> <span class="mi">1</span> <span class="mi">1</span> |
| </pre></div> |
| </div> |
| <p>Arrays having a 0 null count may choose to not allocate the validity |
| bitmap; how this is represented depends on the implementation (for |
| example, a C++ implementation may represent such an “absent” validity |
| bitmap using a NULL pointer). Implementations may choose to always allocate |
| a validity bitmap anyway as a matter of convenience. Consumers of Arrow |
| arrays should be ready to handle those two possibilities.</p> |
| <p>Nested type arrays (except for union types as noted above) have their own |
| top-level validity bitmap and null count, regardless of the null count and |
| valid bits of their child arrays.</p> |
| <p>Array slots which are null are not required to have a particular value; |
| any “masked” memory can have any value and need not be zeroed, though |
| implementations frequently choose to zero memory for null values.</p> |
| </section> |
| <section id="fixed-size-primitive-layout"> |
| <h3>Fixed-size Primitive Layout<a class="headerlink" href="#fixed-size-primitive-layout" title="Permalink to this headline">#</a></h3> |
| <p>A primitive value array represents an array of values each having the |
| same physical slot width typically measured in bytes, though the spec |
| also provides for bit-packed types (e.g. boolean values encoded in |
| bits).</p> |
| <p>Internally, the array contains a contiguous memory buffer whose total |
| size is at least as large as the slot width multiplied by the array |
| length. For bit-packed types, the size is rounded up to the nearest |
| byte.</p> |
| <p>The associated validity bitmap is contiguously allocated (as described |
| above) but does not need to be adjacent in memory to the values |
| buffer.</p> |
| <p><strong>Example Layout: Int32 Array</strong></p> |
| <p>For example a primitive array of int32s:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>Would look like:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span><span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00011101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span><span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-------------|-------------|</span> |
| <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p><strong>Example Layout: Non-null int32 Array</strong></p> |
| <p><code class="docutils literal notranslate"><span class="pre">[1,</span> <span class="pre">2,</span> <span class="pre">3,</span> <span class="pre">4,</span> <span class="pre">8]</span></code> has two possible layouts:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00011111</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span><span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-------------|-------------|</span> |
| <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p>or with the bitmap elided:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span> <span class="mi">5</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span><span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-------------|-------------|</span> |
| <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="variable-size-binary-layout"> |
| <h3>Variable-size Binary Layout<a class="headerlink" href="#variable-size-binary-layout" title="Permalink to this headline">#</a></h3> |
| <p>Each value in this layout consists of 0 or more bytes. While primitive |
| arrays have a single values buffer, variable-size binary have an |
| <strong>offsets</strong> buffer and <strong>data</strong> buffer.</p> |
| <p>The offsets buffer contains <cite>length + 1</cite> signed integers (either |
| 32-bit or 64-bit, depending on the logical type), which encode the |
| start position of each slot in the data buffer. The length of the |
| value in each slot is computed using the difference between the offset |
| at that slot’s index and the subsequent offset. For example, the |
| position and length of slot j is computed as:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">slot_position</span> <span class="o">=</span> <span class="n">offsets</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> |
| <span class="n">slot_length</span> <span class="o">=</span> <span class="n">offsets</span><span class="p">[</span><span class="n">j</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="n">offsets</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> <span class="o">//</span> <span class="p">(</span><span class="k">for</span> <span class="mi">0</span> <span class="o"><=</span> <span class="n">j</span> <span class="o"><</span> <span class="n">length</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>It should be noted that a null value may have a positive slot length. |
| That is, a null value may occupy a <strong>non-empty</strong> memory space in the data |
| buffer. When this is true, the content of the corresponding memory space |
| is undefined.</p> |
| <p>Offsets must be monotonically increasing, that is <code class="docutils literal notranslate"><span class="pre">offsets[j+1]</span> <span class="pre">>=</span> <span class="pre">offsets[j]</span></code> |
| for <code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre"><=</span> <span class="pre">j</span> <span class="pre"><</span> <span class="pre">length</span></code>, even for null slots. This property ensures the |
| location for all values is valid and well defined.</p> |
| <p>Generally the first slot in the offsets array is 0, and the last slot |
| is the length of the values array. When serializing this layout, we |
| recommend normalizing the offsets to start at 0.</p> |
| </section> |
| <section id="variable-size-list-layout"> |
| <h3>Variable-size List Layout<a class="headerlink" href="#variable-size-list-layout" title="Permalink to this headline">#</a></h3> |
| <p>List is a nested type which is semantically similar to variable-size |
| binary. It is defined by two buffers, a validity bitmap and an offsets |
| buffer, and a child array. The offsets are the same as in the |
| variable-size binary case, and both 32-bit and 64-bit signed integer |
| offsets are supported options for the offsets. Rather than referencing |
| an additional data buffer, instead these offsets reference the child |
| array.</p> |
| <p>Similar to the layout of variable-size binary, a null value may |
| correspond to a <strong>non-empty</strong> segment in the child array. When this is |
| true, the content of the corresponding segment can be arbitrary.</p> |
| <p>A list type is specified like <code class="docutils literal notranslate"><span class="pre">List<T></span></code>, where <code class="docutils literal notranslate"><span class="pre">T</span></code> is any type |
| (primitive or nested). In these examples we use 32-bit offsets where |
| the 64-bit offset version would be denoted by <code class="docutils literal notranslate"><span class="pre">LargeList<T></span></code>.</p> |
| <p><strong>Example Layout: ``List<Int8>`` Array</strong></p> |
| <p>We illustrate an example of <code class="docutils literal notranslate"><span class="pre">List<Int8></span></code> with length 4 having values:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[[</span><span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span><span class="p">],</span> <span class="n">null</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span><span class="p">],</span> <span class="p">[]]</span> |
| </pre></div> |
| </div> |
| <p>will have the following representation:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00001101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Offsets</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int32</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-------------|-------------|</span> |
| <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">array</span> <span class="p">(</span><span class="n">Int8array</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int8</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">6</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">7</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------------------------|-------------|</span> |
| <span class="o">|</span> <span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p><strong>Example Layout: ``List<List<Int8>>``</strong></p> |
| <p><code class="docutils literal notranslate"><span class="pre">[[[1,</span> <span class="pre">2],</span> <span class="pre">[3,</span> <span class="pre">4]],</span> <span class="pre">[[5,</span> <span class="pre">6,</span> <span class="pre">7],</span> <span class="pre">null,</span> <span class="pre">[8]],</span> <span class="pre">[[9,</span> <span class="pre">10]]]</span></code></p> |
| <p>will be represented as follows:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>* Length 3 |
| * Nulls count: 0 |
| * Validity bitmap buffer: Not required |
| * Offsets buffer (int32) |
| |
| | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-63 | |
| |------------|------------|------------|-------------|-------------| |
| | 0 | 2 | 5 | 6 | unspecified | |
| |
| * Values array (`List<Int8>`) |
| * Length: 6, Null count: 1 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-------------| |
| | 00110111 | 0 (padding) | |
| |
| * Offsets buffer (int32) |
| |
| | Bytes 0-27 | Bytes 28-63 | |
| |----------------------|-------------| |
| | 0, 2, 4, 7, 7, 8, 10 | unspecified | |
| |
| * Values array (Int8): |
| * Length: 10, Null count: 0 |
| * Validity bitmap buffer: Not required |
| |
| | Bytes 0-9 | Bytes 10-63 | |
| |-------------------------------|-------------| |
| | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 | unspecified | |
| </pre></div> |
| </div> |
| </section> |
| <section id="fixed-size-list-layout"> |
| <h3>Fixed-Size List Layout<a class="headerlink" href="#fixed-size-list-layout" title="Permalink to this headline">#</a></h3> |
| <p>Fixed-Size List is a nested type in which each array slot contains a |
| fixed-size sequence of values all having the same type.</p> |
| <p>A fixed size list type is specified like <code class="docutils literal notranslate"><span class="pre">FixedSizeList<T>[N]</span></code>, |
| where <code class="docutils literal notranslate"><span class="pre">T</span></code> is any type (primitive or nested) and <code class="docutils literal notranslate"><span class="pre">N</span></code> is a 32-bit |
| signed integer representing the length of the lists.</p> |
| <p>A fixed size list array is represented by a values array, which is a |
| child array of type T. T may also be a nested type. The value in slot |
| <code class="docutils literal notranslate"><span class="pre">j</span></code> of a fixed size list array is stored in an <code class="docutils literal notranslate"><span class="pre">N</span></code>-long slice of |
| the values array, starting at an offset of <code class="docutils literal notranslate"><span class="pre">j</span> <span class="pre">*</span> <span class="pre">N</span></code>.</p> |
| <p><strong>Example Layout: ``FixedSizeList<byte>[4]`` Array</strong></p> |
| <p>Here we illustrate <code class="docutils literal notranslate"><span class="pre">FixedSizeList<byte>[4]</span></code>.</p> |
| <p>For an array of length 4 with respective values:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[[</span><span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">12</span><span class="p">],</span> <span class="n">null</span><span class="p">,</span> <span class="p">[</span><span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">25</span><span class="p">],</span> <span class="p">[</span><span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]]</span> |
| </pre></div> |
| </div> |
| <p>will have the following representation:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00001101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">array</span> <span class="p">(</span><span class="n">byte</span> <span class="n">array</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">16</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> |
| <span class="o">|-----------------|-------------|---------------------------------|</span> |
| <span class="o">|</span> <span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">12</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> <span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">25</span><span class="p">,</span> <span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="struct-layout"> |
| <h3>Struct Layout<a class="headerlink" href="#struct-layout" title="Permalink to this headline">#</a></h3> |
| <p>A struct is a nested type parameterized by an ordered sequence of |
| types (which can all be distinct), called its fields. Each field must |
| have a UTF8-encoded name, and these field names are part of the type |
| metadata.</p> |
| <p>Physically, a struct array has one child array for each field. The |
| child arrays are independent and need not be adjacent to each other in |
| memory. A struct array also has a validity bitmap to encode top-level |
| validity information.</p> |
| <p>For example, the struct (field names shown here as strings for illustration |
| purposes):</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Struct</span> <span class="o"><</span> |
| <span class="n">name</span><span class="p">:</span> <span class="n">VarBinary</span> |
| <span class="n">age</span><span class="p">:</span> <span class="n">Int32</span> |
| <span class="o">></span> |
| </pre></div> |
| </div> |
| <p>has two child arrays, one <code class="docutils literal notranslate"><span class="pre">VarBinary</span></code> array (using variable-size binary |
| layout) and one 4-byte primitive value array having <code class="docutils literal notranslate"><span class="pre">Int32</span></code> logical |
| type.</p> |
| <p><strong>Example Layout: ``Struct<VarBinary, Int32>``</strong></p> |
| <p>The layout for <code class="docutils literal notranslate"><span class="pre">[{'joe',</span> <span class="pre">1},</span> <span class="pre">{null,</span> <span class="pre">2},</span> <span class="pre">null,</span> <span class="pre">{'mark',</span> <span class="pre">4}]</span></code> would be:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>* Length: 4, Null count: 1 |
| * Validity bitmap buffer: |
| |
| |Byte 0 (validity bitmap) | Bytes 1-63 | |
| |-------------------------|-----------------------| |
| | 00001011 | 0 (padding) | |
| |
| * Children arrays: |
| * field-0 array (`VarBinary`): |
| * Length: 4, Null count: 2 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00001001 | 0 (padding) | |
| |
| * Offsets buffer: |
| |
| | Bytes 0-19 | |
| |----------------| |
| | 0, 3, 3, 3, 7 | |
| |
| * Values array: |
| * Length: 7, Null count: 0 |
| * Validity bitmap buffer: Not required |
| |
| * Value buffer: |
| |
| | Bytes 0-6 | |
| |----------------| |
| | joemark | |
| |
| * field-1 array (int32 array): |
| * Length: 4, Null count: 1 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00001011 | 0 (padding) | |
| |
| * Value Buffer: |
| |
| |Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-63 | |
| |------------|-------------|-------------|-------------|-------------| |
| | 1 | 2 | unspecified | 4 | unspecified | |
| </pre></div> |
| </div> |
| <section id="struct-validity"> |
| <h4>Struct Validity<a class="headerlink" href="#struct-validity" title="Permalink to this headline">#</a></h4> |
| <p>A struct array has its own validity bitmap that is independent of its |
| child arrays’ validity bitmaps. The validity bitmap for the struct |
| array might indicate a null when one or more of its child arrays has |
| a non-null value in its corresponding slot; or conversely, a child |
| array might have a null in its validity bitmap while the struct array’s |
| validity bitmap shows a non-null value.</p> |
| <p>Therefore, to know whether a particular child entry is valid, one must |
| take the logical AND of the corresponding bits in the two validity bitmaps |
| (the struct array’s and the child array’s).</p> |
| <p>This is illustrated in the example above, the child arrays have valid entries |
| for the null struct but they are “hidden” by the struct array’s validity |
| bitmap. However, when treated independently, corresponding entries of the |
| children array will be non-null.</p> |
| </section> |
| </section> |
| <section id="union-layout"> |
| <h3>Union Layout<a class="headerlink" href="#union-layout" title="Permalink to this headline">#</a></h3> |
| <p>A union is defined by an ordered sequence of types; each slot in the |
| union can have a value chosen from these types. The types are named |
| like a struct’s fields, and the names are part of the type metadata.</p> |
| <p>Unlike other data types, unions do not have their own validity bitmap. Instead, |
| the nullness of each slot is determined exclusively by the child arrays which |
| are composed to create the union.</p> |
| <p>We define two distinct union types, “dense” and “sparse”, that are |
| optimized for different use cases.</p> |
| <section id="dense-union"> |
| <h4>Dense Union<a class="headerlink" href="#dense-union" title="Permalink to this headline">#</a></h4> |
| <p>Dense union represents a mixed-type array with 5 bytes of overhead for |
| each value. Its physical layout is as follows:</p> |
| <ul class="simple"> |
| <li><p>One child array for each type</p></li> |
| <li><p>Types buffer: A buffer of 8-bit signed integers. Each type in the |
| union has a corresponding type id whose values are found in this |
| buffer. A union with more than 127 possible types can be modeled as |
| a union of unions.</p></li> |
| <li><p>Offsets buffer: A buffer of signed Int32 values indicating the |
| relative offset into the respective child array for the type in a |
| given slot. The respective offsets for each child value array must |
| be in order / increasing.</p></li> |
| </ul> |
| <p><strong>Example Layout: ``DenseUnion<f: Float32, i: Int32>``</strong></p> |
| <p>For the union array:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="n">f</span><span class="o">=</span><span class="mf">1.2</span><span class="p">},</span> <span class="n">null</span><span class="p">,</span> <span class="p">{</span><span class="n">f</span><span class="o">=</span><span class="mf">3.4</span><span class="p">},</span> <span class="p">{</span><span class="n">i</span><span class="o">=</span><span class="mi">5</span><span class="p">}]</span> |
| </pre></div> |
| </div> |
| <p>will have the following layout:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Types</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span><span class="n">Byte</span> <span class="mi">0</span> <span class="o">|</span> <span class="n">Byte</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Byte</span> <span class="mi">2</span> <span class="o">|</span> <span class="n">Byte</span> <span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|---------|-------------|----------|----------|-------------|</span> |
| <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Offset</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span><span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|----------|-------------|------------|-------------|-------------|</span> |
| <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Children</span> <span class="n">arrays</span><span class="p">:</span> |
| <span class="o">*</span> <span class="n">Field</span><span class="o">-</span><span class="mi">0</span> <span class="n">array</span> <span class="p">(</span><span class="n">f</span><span class="p">:</span> <span class="n">Float32</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="mi">00000101</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|----------------|-------------|</span> |
| <span class="o">|</span> <span class="mf">1.2</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mf">3.4</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| |
| |
| <span class="o">*</span> <span class="n">Field</span><span class="o">-</span><span class="mi">1</span> <span class="n">array</span> <span class="p">(</span><span class="n">i</span><span class="p">:</span> <span class="n">Int32</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-----------|-------------|</span> |
| <span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="sparse-union"> |
| <h4>Sparse Union<a class="headerlink" href="#sparse-union" title="Permalink to this headline">#</a></h4> |
| <p>A sparse union has the same structure as a dense union, with the omission of |
| the offsets array. In this case, the child arrays are each equal in length to |
| the length of the union.</p> |
| <p>While a sparse union may use significantly more space compared with a |
| dense union, it has some advantages that may be desirable in certain |
| use cases:</p> |
| <ul class="simple"> |
| <li><p>A sparse union is more amenable to vectorized expression evaluation in some use cases.</p></li> |
| <li><p>Equal-length arrays can be interpreted as a union by only defining the types array.</p></li> |
| </ul> |
| <p><strong>Example layout: ``SparseUnion<i: Int32, f: Float32, s: VarBinary>``</strong></p> |
| <p>For the union array:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="n">i</span><span class="o">=</span><span class="mi">5</span><span class="p">},</span> <span class="p">{</span><span class="n">f</span><span class="o">=</span><span class="mf">1.2</span><span class="p">},</span> <span class="p">{</span><span class="n">s</span><span class="o">=</span><span class="s1">'joe'</span><span class="p">},</span> <span class="p">{</span><span class="n">f</span><span class="o">=</span><span class="mf">3.4</span><span class="p">},</span> <span class="p">{</span><span class="n">i</span><span class="o">=</span><span class="mi">4</span><span class="p">},</span> <span class="p">{</span><span class="n">s</span><span class="o">=</span><span class="s1">'mark'</span><span class="p">}]</span> |
| </pre></div> |
| </div> |
| <p>will have the following layout:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>* Length: 6, Null count: 0 |
| * Types buffer: |
| |
| | Byte 0 | Byte 1 | Byte 2 | Byte 3 | Byte 4 | Byte 5 | Bytes 6-63 | |
| |------------|-------------|-------------|-------------|-------------|--------------|-----------------------| |
| | 0 | 1 | 2 | 1 | 0 | 2 | unspecified (padding) | |
| |
| * Children arrays: |
| |
| * i (Int32): |
| * Length: 6, Null count: 4 |
| * Validity bitmap buffer: |
| |
| |Byte 0 (validity bitmap) | Bytes 1-63 | |
| |-------------------------|-----------------------| |
| |00010001 | 0 (padding) | |
| |
| * Value buffer: |
| |
| |Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-63 | |
| |------------|-------------|-------------|-------------|-------------|--------------|-----------------------| |
| | 5 | unspecified | unspecified | unspecified | 4 | unspecified | unspecified (padding) | |
| |
| * f (Float32): |
| * Length: 6, Null count: 4 |
| * Validity bitmap buffer: |
| |
| |Byte 0 (validity bitmap) | Bytes 1-63 | |
| |-------------------------|-----------------------| |
| | 00001010 | 0 (padding) | |
| |
| * Value buffer: |
| |
| |Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-63 | |
| |-------------|-------------|-------------|-------------|-------------|--------------|-----------------------| |
| | unspecified | 1.2 | unspecified | 3.4 | unspecified | unspecified | unspecified (padding) | |
| |
| * s (`VarBinary`) |
| * Length: 6, Null count: 4 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00100100 | 0 (padding) | |
| |
| * Offsets buffer (Int32) |
| |
| | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-27 | Bytes 28-63 | |
| |------------|-------------|-------------|-------------|-------------|-------------|-------------|-------------| |
| | 0 | 0 | 0 | 3 | 3 | 3 | 7 | unspecified | |
| |
| * Values array (VarBinary): |
| * Length: 7, Null count: 0 |
| * Validity bitmap buffer: Not required |
| |
| | Bytes 0-6 | Bytes 7-63 | |
| |------------|-----------------------| |
| | joemark | unspecified (padding) | |
| </pre></div> |
| </div> |
| <p>Only the slot in the array corresponding to the type index is considered. All |
| “unselected” values are ignored and could be any semantically correct array |
| value.</p> |
| </section> |
| </section> |
| <section id="null-layout"> |
| <h3>Null Layout<a class="headerlink" href="#null-layout" title="Permalink to this headline">#</a></h3> |
| <p>We provide a simplified memory-efficient layout for the Null data type |
| where all values are null. In this case no memory buffers are |
| allocated.</p> |
| </section> |
| <section id="dictionary-encoded-layout"> |
| <span id="id1"></span><h3>Dictionary-encoded Layout<a class="headerlink" href="#dictionary-encoded-layout" title="Permalink to this headline">#</a></h3> |
| <p>Dictionary encoding is a data representation technique to represent |
| values by integers referencing a <strong>dictionary</strong> usually consisting of |
| unique values. It can be effective when you have data with many |
| repeated values.</p> |
| <p>Any array can be dictionary-encoded. The dictionary is stored as an optional |
| property of an array. When a field is dictionary encoded, the values are |
| represented by an array of non-negative integers representing the index of the |
| value in the dictionary. The memory layout for a dictionary-encoded array is |
| the same as that of a primitive integer layout. The dictionary is handled as a |
| separate columnar array with its own respective layout.</p> |
| <p>As an example, you could have the following data:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">type</span><span class="p">:</span> <span class="n">VarBinary</span> |
| |
| <span class="p">[</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>In dictionary-encoded form, this could appear as:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">data</span> <span class="n">VarBinary</span> <span class="p">(</span><span class="n">dictionary</span><span class="o">-</span><span class="n">encoded</span><span class="p">)</span> |
| <span class="n">index_type</span><span class="p">:</span> <span class="n">Int32</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span> |
| |
| <span class="n">dictionary</span> |
| <span class="nb">type</span><span class="p">:</span> <span class="n">VarBinary</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>Note that a dictionary is permitted to contain duplicate values or |
| nulls:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">data</span> <span class="n">VarBinary</span> <span class="p">(</span><span class="n">dictionary</span><span class="o">-</span><span class="n">encoded</span><span class="p">)</span> |
| <span class="n">index_type</span><span class="p">:</span> <span class="n">Int32</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span> |
| |
| <span class="n">dictionary</span> |
| <span class="nb">type</span><span class="p">:</span> <span class="n">VarBinary</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">,</span> <span class="s1">'foo'</span><span class="p">,</span> <span class="n">null</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>The null count of such arrays is dictated only by the validity bitmap |
| of its indices, irrespective of any null values in the dictionary.</p> |
| <p>Since unsigned integers can be more difficult to work with in some cases |
| (e.g. in the JVM), we recommend preferring signed integers over unsigned |
| integers for representing dictionary indices. Additionally, we recommend |
| avoiding using 64-bit unsigned integer indices unless they are required by an |
| application.</p> |
| <p>We discuss dictionary encoding as it relates to serialization further |
| below.</p> |
| </section> |
| <section id="buffer-listing-for-each-layout"> |
| <h3>Buffer Listing for Each Layout<a class="headerlink" href="#buffer-listing-for-each-layout" title="Permalink to this headline">#</a></h3> |
| <p>For the avoidance of ambiguity, we provide listing the order and type |
| of memory buffers for each layout.</p> |
| <table class="colwidths-given table" id="id2"> |
| <caption><span class="caption-text">Buffer Layouts</span><a class="headerlink" href="#id2" title="Permalink to this table">#</a></caption> |
| <colgroup> |
| <col style="width: 33%" /> |
| <col style="width: 22%" /> |
| <col style="width: 22%" /> |
| <col style="width: 22%" /> |
| </colgroup> |
| <thead> |
| <tr class="row-odd"><th class="head"><p>Layout Type</p></th> |
| <th class="head"><p>Buffer 0</p></th> |
| <th class="head"><p>Buffer 1</p></th> |
| <th class="head"><p>Buffer 2</p></th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="row-even"><td><p>Primitive</p></td> |
| <td><p>validity</p></td> |
| <td><p>data</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Variable Binary</p></td> |
| <td><p>validity</p></td> |
| <td><p>offsets</p></td> |
| <td><p>data</p></td> |
| </tr> |
| <tr class="row-even"><td><p>List</p></td> |
| <td><p>validity</p></td> |
| <td><p>offsets</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Fixed-size List</p></td> |
| <td><p>validity</p></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Struct</p></td> |
| <td><p>validity</p></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Sparse Union</p></td> |
| <td><p>type ids</p></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Dense Union</p></td> |
| <td><p>type ids</p></td> |
| <td><p>offsets</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Null</p></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Dictionary-encoded</p></td> |
| <td><p>validity</p></td> |
| <td><p>data (indices)</p></td> |
| <td></td> |
| </tr> |
| </tbody> |
| </table> |
| </section> |
| </section> |
| <section id="logical-types"> |
| <h2>Logical Types<a class="headerlink" href="#logical-types" title="Permalink to this headline">#</a></h2> |
| <p>The <a class="reference external" href="https://github.com/apache/arrow/blob/master/format/Schema.fbs">Schema.fbs</a> defines built-in logical types supported by the |
| Arrow columnar format. Each logical type uses one of the above |
| physical layouts. Nested logical types may have different physical |
| layouts depending on the particular realization of the type.</p> |
| <p>We do not go into detail about the logical types definitions in this |
| document as we consider <a class="reference external" href="https://github.com/apache/arrow/blob/master/format/Schema.fbs">Schema.fbs</a> to be authoritative.</p> |
| </section> |
| <section id="serialization-and-interprocess-communication-ipc"> |
| <span id="format-ipc"></span><h2>Serialization and Interprocess Communication (IPC)<a class="headerlink" href="#serialization-and-interprocess-communication-ipc" title="Permalink to this headline">#</a></h2> |
| <p>The primitive unit of serialized data in the columnar format is the |
| “record batch”. Semantically, a record batch is an ordered collection |
| of arrays, known as its <strong>fields</strong>, each having the same length as one |
| another but potentially different data types. A record batch’s field |
| names and types collectively form the batch’s <strong>schema</strong>.</p> |
| <p>In this section we define a protocol for serializing record batches |
| into a stream of binary payloads and reconstructing record batches |
| from these payloads without need for memory copying.</p> |
| <p>The columnar IPC protocol utilizes a one-way stream of binary messages |
| of these types:</p> |
| <ul class="simple"> |
| <li><p>Schema</p></li> |
| <li><p>RecordBatch</p></li> |
| <li><p>DictionaryBatch</p></li> |
| </ul> |
| <p>We specify a so-called <em>encapsulated IPC message</em> format which |
| includes a serialized Flatbuffer type along with an optional message |
| body. We define this message format before describing how to serialize |
| each constituent IPC message type.</p> |
| <section id="encapsulated-message-format"> |
| <h3>Encapsulated message format<a class="headerlink" href="#encapsulated-message-format" title="Permalink to this headline">#</a></h3> |
| <p>For simple streaming and file-based serialization, we define a |
| “encapsulated” message format for interprocess communication. Such |
| messages can be “deserialized” into in-memory Arrow array objects by |
| examining only the message metadata without any need to copy or move |
| any of the actual data.</p> |
| <p>The encapsulated binary message format is as follows:</p> |
| <ul class="simple"> |
| <li><p>A 32-bit continuation indicator. The value <code class="docutils literal notranslate"><span class="pre">0xFFFFFFFF</span></code> indicates |
| a valid message. This component was introduced in version 0.15.0 in |
| part to address the 8-byte alignment requirement of Flatbuffers</p></li> |
| <li><p>A 32-bit little-endian length prefix indicating the metadata size</p></li> |
| <li><p>The message metadata as using the <code class="docutils literal notranslate"><span class="pre">Message</span></code> type defined in |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/format/Message.fbs">Message.fbs</a></p></li> |
| <li><p>Padding bytes to an 8-byte boundary</p></li> |
| <li><p>The message body, whose length must be a multiple of 8 bytes</p></li> |
| </ul> |
| <p>Schematically, we have:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">continuation</span><span class="p">:</span> <span class="mh">0xFFFFFFFF</span><span class="o">></span> |
| <span class="o"><</span><span class="n">metadata_size</span><span class="p">:</span> <span class="n">int32</span><span class="o">></span> |
| <span class="o"><</span><span class="n">metadata_flatbuffer</span><span class="p">:</span> <span class="nb">bytes</span><span class="o">></span> |
| <span class="o"><</span><span class="n">padding</span><span class="o">></span> |
| <span class="o"><</span><span class="n">message</span> <span class="n">body</span><span class="o">></span> |
| </pre></div> |
| </div> |
| <p>The complete serialized message must be a multiple of 8 bytes so that messages |
| can be relocated between streams. Otherwise the amount of padding between the |
| metadata and the message body could be non-deterministic.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">metadata_size</span></code> includes the size of the <code class="docutils literal notranslate"><span class="pre">Message</span></code> plus |
| padding. The <code class="docutils literal notranslate"><span class="pre">metadata_flatbuffer</span></code> contains a serialized <code class="docutils literal notranslate"><span class="pre">Message</span></code> |
| Flatbuffer value, which internally includes:</p> |
| <ul class="simple"> |
| <li><p>A version number</p></li> |
| <li><p>A particular message value (one of <code class="docutils literal notranslate"><span class="pre">Schema</span></code>, <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>, or |
| <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code>)</p></li> |
| <li><p>The size of the message body</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> field for any application-supplied metadata</p></li> |
| </ul> |
| <p>When read from an input stream, generally the <code class="docutils literal notranslate"><span class="pre">Message</span></code> metadata is |
| initially parsed and validated to obtain the body size. Then the body |
| can be read.</p> |
| </section> |
| <section id="schema-message"> |
| <h3>Schema message<a class="headerlink" href="#schema-message" title="Permalink to this headline">#</a></h3> |
| <p>The Flatbuffers files <a class="reference external" href="https://github.com/apache/arrow/blob/master/format/Schema.fbs">Schema.fbs</a> contains the definitions for all |
| built-in logical data types and the <code class="docutils literal notranslate"><span class="pre">Schema</span></code> metadata type which |
| represents the schema of a given record batch. A schema consists of |
| an ordered sequence of fields, each having a name and type. A |
| serialized <code class="docutils literal notranslate"><span class="pre">Schema</span></code> does not contain any data buffers, only type |
| metadata.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">Field</span></code> Flatbuffers type contains the metadata for a single |
| array. This includes:</p> |
| <ul class="simple"> |
| <li><p>The field’s name</p></li> |
| <li><p>The field’s logical type</p></li> |
| <li><p>Whether the field is semantically nullable. While this has no |
| bearing on the array’s physical layout, many systems distinguish |
| nullable and non-nullable fields and we want to allow them to |
| preserve this metadata to enable faithful schema round trips.</p></li> |
| <li><p>A collection of child <code class="docutils literal notranslate"><span class="pre">Field</span></code> values, for nested types</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">dictionary</span></code> property indicating whether the field is |
| dictionary-encoded or not. If it is, a dictionary “id” is assigned |
| to allow matching a subsequent dictionary IPC message with the |
| appropriate field.</p></li> |
| </ul> |
| <p>We additionally provide both schema-level and field-level |
| <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> attributes allowing for systems to insert their |
| own application defined metadata to customize behavior.</p> |
| </section> |
| <section id="recordbatch-message"> |
| <h3>RecordBatch message<a class="headerlink" href="#recordbatch-message" title="Permalink to this headline">#</a></h3> |
| <p>A RecordBatch message contains the actual data buffers corresponding |
| to the physical memory layout determined by a schema. The metadata for |
| this message provides the location and size of each buffer, permitting |
| Array data structures to be reconstructed using pointer arithmetic and |
| thus no memory copying.</p> |
| <p>The serialized form of the record batch is the following:</p> |
| <ul class="simple"> |
| <li><p>The <code class="docutils literal notranslate"><span class="pre">data</span> <span class="pre">header</span></code>, defined as the <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> type in |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/format/Message.fbs">Message.fbs</a>.</p></li> |
| <li><p>The <code class="docutils literal notranslate"><span class="pre">body</span></code>, a flat sequence of memory buffers written end-to-end |
| with appropriate padding to ensure a minimum of 8-byte alignment</p></li> |
| </ul> |
| <p>The data header contains the following:</p> |
| <ul class="simple"> |
| <li><p>The length and null count for each flattened field in the record |
| batch</p></li> |
| <li><p>The memory offset and length of each constituent <code class="docutils literal notranslate"><span class="pre">Buffer</span></code> in the |
| record batch’s body</p></li> |
| </ul> |
| <p>Fields and buffers are flattened by a pre-order depth-first traversal |
| of the fields in the record batch. For example, let’s consider the |
| schema</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">col1</span><span class="p">:</span> <span class="n">Struct</span><span class="o"><</span><span class="n">a</span><span class="p">:</span> <span class="n">Int32</span><span class="p">,</span> <span class="n">b</span><span class="p">:</span> <span class="n">List</span><span class="o"><</span><span class="n">item</span><span class="p">:</span> <span class="n">Int64</span><span class="o">></span><span class="p">,</span> <span class="n">c</span><span class="p">:</span> <span class="n">Float64</span><span class="o">></span> |
| <span class="n">col2</span><span class="p">:</span> <span class="n">Utf8</span> |
| </pre></div> |
| </div> |
| <p>The flattened version of this is:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">FieldNode</span> <span class="mi">0</span><span class="p">:</span> <span class="n">Struct</span> <span class="n">name</span><span class="o">=</span><span class="s1">'col1'</span> |
| <span class="n">FieldNode</span> <span class="mi">1</span><span class="p">:</span> <span class="n">Int32</span> <span class="n">name</span><span class="o">=</span><span class="s1">'a'</span> |
| <span class="n">FieldNode</span> <span class="mi">2</span><span class="p">:</span> <span class="n">List</span> <span class="n">name</span><span class="o">=</span><span class="s1">'b'</span> |
| <span class="n">FieldNode</span> <span class="mi">3</span><span class="p">:</span> <span class="n">Int64</span> <span class="n">name</span><span class="o">=</span><span class="s1">'item'</span> |
| <span class="n">FieldNode</span> <span class="mi">4</span><span class="p">:</span> <span class="n">Float64</span> <span class="n">name</span><span class="o">=</span><span class="s1">'c'</span> |
| <span class="n">FieldNode</span> <span class="mi">5</span><span class="p">:</span> <span class="n">Utf8</span> <span class="n">name</span><span class="o">=</span><span class="s1">'col2'</span> |
| </pre></div> |
| </div> |
| <p>For the buffers produced, we would have the following (refer to the |
| table above):</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">buffer</span> <span class="mi">0</span><span class="p">:</span> <span class="n">field</span> <span class="mi">0</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">1</span><span class="p">:</span> <span class="n">field</span> <span class="mi">1</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">2</span><span class="p">:</span> <span class="n">field</span> <span class="mi">1</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">3</span><span class="p">:</span> <span class="n">field</span> <span class="mi">2</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">4</span><span class="p">:</span> <span class="n">field</span> <span class="mi">2</span> <span class="n">offsets</span> |
| <span class="n">buffer</span> <span class="mi">5</span><span class="p">:</span> <span class="n">field</span> <span class="mi">3</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">6</span><span class="p">:</span> <span class="n">field</span> <span class="mi">3</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">7</span><span class="p">:</span> <span class="n">field</span> <span class="mi">4</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">8</span><span class="p">:</span> <span class="n">field</span> <span class="mi">4</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">9</span><span class="p">:</span> <span class="n">field</span> <span class="mi">5</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">10</span><span class="p">:</span> <span class="n">field</span> <span class="mi">5</span> <span class="n">offsets</span> |
| <span class="n">buffer</span> <span class="mi">11</span><span class="p">:</span> <span class="n">field</span> <span class="mi">5</span> <span class="n">data</span> |
| </pre></div> |
| </div> |
| <p>The <code class="docutils literal notranslate"><span class="pre">Buffer</span></code> Flatbuffers value describes the location and size of a |
| piece of memory. Generally these are interpreted relative to the |
| <strong>encapsulated message format</strong> defined below.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">size</span></code> field of <code class="docutils literal notranslate"><span class="pre">Buffer</span></code> is not required to account for padding |
| bytes. Since this metadata can be used to communicate in-memory pointer |
| addresses between libraries, it is recommended to set <code class="docutils literal notranslate"><span class="pre">size</span></code> to the actual |
| memory size rather than the padded size.</p> |
| </section> |
| <section id="byte-order-endianness"> |
| <h3>Byte Order (<a class="reference external" href="https://en.wikipedia.org/wiki/Endianness">Endianness</a>)<a class="headerlink" href="#byte-order-endianness" title="Permalink to this headline">#</a></h3> |
| <p>The Arrow format is little endian by default.</p> |
| <p>Serialized Schema metadata has an endianness field indicating |
| endianness of RecordBatches. Typically this is the endianness of the |
| system where the RecordBatch was generated. The main use case is |
| exchanging RecordBatches between systems with the same Endianness. At |
| first we will return an error when trying to read a Schema with an |
| endianness that does not match the underlying system. The reference |
| implementation is focused on Little Endian and provides tests for |
| it. Eventually we may provide automatic conversion via byte swapping.</p> |
| </section> |
| <section id="ipc-streaming-format"> |
| <h3>IPC Streaming Format<a class="headerlink" href="#ipc-streaming-format" title="Permalink to this headline">#</a></h3> |
| <p>We provide a streaming protocol or “format” for record batches. It is |
| presented as a sequence of encapsulated messages, each of which |
| follows the format above. The schema comes first in the stream, and it |
| is the same for all of the record batches that follow. If any fields |
| in the schema are dictionary-encoded, one or more <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code> |
| messages will be included. <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code> and <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> |
| messages may be interleaved, but before any dictionary key is used in |
| a <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> it should be defined in a <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code>.</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">SCHEMA</span><span class="o">></span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="n">k</span> <span class="o">-</span> <span class="mi">1</span><span class="o">></span> |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">0</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="n">x</span> <span class="n">DELTA</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="n">y</span> <span class="n">DELTA</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="o">></span> |
| <span class="o"><</span><span class="n">EOS</span> <span class="p">[</span><span class="n">optional</span><span class="p">]:</span> <span class="mh">0xFFFFFFFF</span> <span class="mh">0x00000000</span><span class="o">></span> |
| </pre></div> |
| </div> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>An edge-case for interleaved dictionary and record batches occurs |
| when the record batches contain dictionary encoded arrays that are |
| completely null. In this case, the dictionary for the encoded column might |
| appear after the first record batch.</p> |
| </div> |
| <p>When a stream reader implementation is reading a stream, after each |
| message, it may read the next 8 bytes to determine both if the stream |
| continues and the size of the message metadata that follows. Once the |
| message flatbuffer is read, you can then read the message body.</p> |
| <p>The stream writer can signal end-of-stream (EOS) either by writing 8 bytes |
| containing the 4-byte continuation indicator (<code class="docutils literal notranslate"><span class="pre">0xFFFFFFFF</span></code>) followed by 0 |
| metadata length (<code class="docutils literal notranslate"><span class="pre">0x00000000</span></code>) or closing the stream interface. We |
| recommend the “.arrows” file extension for the streaming format although |
| in many cases these streams will not ever be stored as files.</p> |
| </section> |
| <section id="ipc-file-format"> |
| <h3>IPC File Format<a class="headerlink" href="#ipc-file-format" title="Permalink to this headline">#</a></h3> |
| <p>We define a “file format” supporting random access that is an extension of |
| the stream format. The file starts and ends with a magic string <code class="docutils literal notranslate"><span class="pre">ARROW1</span></code> |
| (plus padding). What follows in the file is identical to the stream format. |
| At the end of the file, we write a <em>footer</em> containing a redundant copy of |
| the schema (which is a part of the streaming format) plus memory offsets and |
| sizes for each of the data blocks in the file. This enables random access to |
| any record batch in the file. See <a class="reference external" href="https://github.com/apache/arrow/blob/master/format/File.fbs">File.fbs</a> for the precise details of the |
| file footer.</p> |
| <p>Schematically we have:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">magic</span> <span class="n">number</span> <span class="s2">"ARROW1"</span><span class="o">></span> |
| <span class="o"><</span><span class="n">empty</span> <span class="n">padding</span> <span class="nb">bytes</span> <span class="p">[</span><span class="n">to</span> <span class="mi">8</span> <span class="n">byte</span> <span class="n">boundary</span><span class="p">]</span><span class="o">></span> |
| <span class="o"><</span><span class="n">STREAMING</span> <span class="n">FORMAT</span> <span class="k">with</span> <span class="n">EOS</span><span class="o">></span> |
| <span class="o"><</span><span class="n">FOOTER</span><span class="o">></span> |
| <span class="o"><</span><span class="n">FOOTER</span> <span class="n">SIZE</span><span class="p">:</span> <span class="n">int32</span><span class="o">></span> |
| <span class="o"><</span><span class="n">magic</span> <span class="n">number</span> <span class="s2">"ARROW1"</span><span class="o">></span> |
| </pre></div> |
| </div> |
| <p>In the file format, there is no requirement that dictionary keys |
| should be defined in a <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code> before they are used in a |
| <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>, as long as the keys are defined somewhere in the |
| file. Further more, it is invalid to have more than one <strong>non-delta</strong> |
| dictionary batch per dictionary ID (i.e. dictionary replacement is not |
| supported). Delta dictionaries are applied in the order they appear in |
| the file footer. We recommend the “.arrow” extension for files created with |
| this format.</p> |
| </section> |
| <section id="dictionary-messages"> |
| <h3>Dictionary Messages<a class="headerlink" href="#dictionary-messages" title="Permalink to this headline">#</a></h3> |
| <p>Dictionaries are written in the stream and file formats as a sequence of record |
| batches, each having a single field. The complete semantic schema for a |
| sequence of record batches, therefore, consists of the schema along with all of |
| the dictionaries. The dictionary types are found in the schema, so it is |
| necessary to read the schema to first determine the dictionary types so that |
| the dictionaries can be properly interpreted:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">table</span> <span class="n">DictionaryBatch</span> <span class="p">{</span> |
| <span class="nb">id</span><span class="p">:</span> <span class="n">long</span><span class="p">;</span> |
| <span class="n">data</span><span class="p">:</span> <span class="n">RecordBatch</span><span class="p">;</span> |
| <span class="n">isDelta</span><span class="p">:</span> <span class="n">boolean</span> <span class="o">=</span> <span class="n">false</span><span class="p">;</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>The dictionary <code class="docutils literal notranslate"><span class="pre">id</span></code> in the message metadata can be referenced one or more times |
| in the schema, so that dictionaries can even be used for multiple fields. See |
| the <a class="reference internal" href="#dictionary-encoded-layout"><span class="std std-ref">Dictionary-encoded Layout</span></a> section for more about the semantics of |
| dictionary-encoded data.</p> |
| <p>The dictionary <code class="docutils literal notranslate"><span class="pre">isDelta</span></code> flag allows existing dictionaries to be |
| expanded for future record batch materializations. A dictionary batch |
| with <code class="docutils literal notranslate"><span class="pre">isDelta</span></code> set indicates that its vector should be concatenated |
| with those of any previous batches with the same <code class="docutils literal notranslate"><span class="pre">id</span></code>. In a stream |
| which encodes one column, the list of strings <code class="docutils literal notranslate"><span class="pre">["A",</span> <span class="pre">"B",</span> <span class="pre">"C",</span> <span class="pre">"B",</span> |
| <span class="pre">"D",</span> <span class="pre">"C",</span> <span class="pre">"E",</span> <span class="pre">"A"]</span></code>, with a delta dictionary batch could take the |
| form:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">SCHEMA</span><span class="o">></span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="s2">"A"</span> |
| <span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="s2">"B"</span> |
| <span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="s2">"C"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">0</span><span class="o">></span> |
| <span class="mi">0</span> |
| <span class="mi">1</span> |
| <span class="mi">2</span> |
| <span class="mi">1</span> |
| |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span> <span class="n">DELTA</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">3</span><span class="p">)</span> <span class="s2">"D"</span> |
| <span class="p">(</span><span class="mi">4</span><span class="p">)</span> <span class="s2">"E"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">1</span><span class="o">></span> |
| <span class="mi">3</span> |
| <span class="mi">2</span> |
| <span class="mi">4</span> |
| <span class="mi">0</span> |
| <span class="n">EOS</span> |
| </pre></div> |
| </div> |
| <p>Alternatively, if <code class="docutils literal notranslate"><span class="pre">isDelta</span></code> is set to false, then the dictionary |
| replaces the existing dictionary for the same ID. Using the same |
| example as above, an alternate encoding could be:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">SCHEMA</span><span class="o">></span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="s2">"A"</span> |
| <span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="s2">"B"</span> |
| <span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="s2">"C"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">0</span><span class="o">></span> |
| <span class="mi">0</span> |
| <span class="mi">1</span> |
| <span class="mi">2</span> |
| <span class="mi">1</span> |
| |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="s2">"A"</span> |
| <span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="s2">"C"</span> |
| <span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="s2">"D"</span> |
| <span class="p">(</span><span class="mi">3</span><span class="p">)</span> <span class="s2">"E"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">1</span><span class="o">></span> |
| <span class="mi">2</span> |
| <span class="mi">1</span> |
| <span class="mi">3</span> |
| <span class="mi">0</span> |
| <span class="n">EOS</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="custom-application-metadata"> |
| <h3>Custom Application Metadata<a class="headerlink" href="#custom-application-metadata" title="Permalink to this headline">#</a></h3> |
| <p>We provide a <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> field at three levels to provide a |
| mechanism for developers to pass application-specific metadata in |
| Arrow protocol messages. This includes <code class="docutils literal notranslate"><span class="pre">Field</span></code>, <code class="docutils literal notranslate"><span class="pre">Schema</span></code>, and |
| <code class="docutils literal notranslate"><span class="pre">Message</span></code>.</p> |
| <p>The colon symbol <code class="docutils literal notranslate"><span class="pre">:</span></code> is to be used as a namespace separator. It can |
| be used multiple times in a key.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">ARROW</span></code> pattern is a reserved namespace for internal Arrow use |
| in the <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> fields. For example, |
| <code class="docutils literal notranslate"><span class="pre">ARROW:extension:name</span></code>.</p> |
| </section> |
| <section id="extension-types"> |
| <span id="format-metadata-extension-types"></span><h3>Extension Types<a class="headerlink" href="#extension-types" title="Permalink to this headline">#</a></h3> |
| <p>User-defined “extension” types can be defined setting certain |
| <code class="docutils literal notranslate"><span class="pre">KeyValue</span></code> pairs in <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> in the <code class="docutils literal notranslate"><span class="pre">Field</span></code> metadata |
| structure. These extension keys are:</p> |
| <ul class="simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">'ARROW:extension:name'</span></code> for the string name identifying the |
| custom data type. We recommend that you use a “namespace”-style |
| prefix for extension type names to minimize the possibility of |
| conflicts with multiple Arrow readers and writers in the same |
| application. For example, use <code class="docutils literal notranslate"><span class="pre">myorg.name_of_type</span></code> instead of |
| simply <code class="docutils literal notranslate"><span class="pre">name_of_type</span></code></p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">'ARROW:extension:metadata'</span></code> for a serialized representation |
| of the <code class="docutils literal notranslate"><span class="pre">ExtensionType</span></code> necessary to reconstruct the custom type</p></li> |
| </ul> |
| <p>This extension metadata can annotate any of the built-in Arrow logical |
| types. The intent is that an implementation that does not support an |
| extension type can still handle the underlying data. For example a |
| 16-byte UUID value could be embedded in <code class="docutils literal notranslate"><span class="pre">FixedSizeBinary(16)</span></code>, and |
| implementations that do not have this extension type can still work |
| with the underlying binary values and pass along the |
| <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> in subsequent Arrow protocol messages.</p> |
| <p>Extension types may or may not use the |
| <code class="docutils literal notranslate"><span class="pre">'ARROW:extension:metadata'</span></code> field. Let’s consider some example |
| extension types:</p> |
| <ul class="simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">uuid</span></code> represented as <code class="docutils literal notranslate"><span class="pre">FixedSizeBinary(16)</span></code> with empty metadata</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">latitude-longitude</span></code> represented as <code class="docutils literal notranslate"><span class="pre">struct<latitude:</span> <span class="pre">double,</span> |
| <span class="pre">longitude:</span> <span class="pre">double></span></code>, and empty metadata</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">tensor</span></code> (multidimensional array) stored as <code class="docutils literal notranslate"><span class="pre">Binary</span></code> values and |
| having serialized metadata indicating the data type and shape of |
| each value. This could be JSON like <code class="docutils literal notranslate"><span class="pre">{'type':</span> <span class="pre">'int8',</span> <span class="pre">'shape':</span> <span class="pre">[4,</span> |
| <span class="pre">5]}</span></code> for a 4x5 cell tensor.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">trading-time</span></code> represented as <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> with serialized |
| metadata indicating the market trading calendar the data corresponds |
| to</p></li> |
| </ul> |
| </section> |
| </section> |
| <section id="implementation-guidelines"> |
| <h2>Implementation guidelines<a class="headerlink" href="#implementation-guidelines" title="Permalink to this headline">#</a></h2> |
| <p>An execution engine (or framework, or UDF executor, or storage engine, |
| etc) can implement only a subset of the Arrow spec and/or extend it |
| given the following constraints:</p> |
| <section id="implementing-a-subset-the-spec"> |
| <h3>Implementing a subset the spec<a class="headerlink" href="#implementing-a-subset-the-spec" title="Permalink to this headline">#</a></h3> |
| <ul class="simple"> |
| <li><p><strong>If only producing (and not consuming) arrow vectors</strong>: Any subset |
| of the vector spec and the corresponding metadata can be implemented.</p></li> |
| <li><p><strong>If consuming and producing vectors</strong>: There is a minimal subset of |
| vectors to be supported. Production of a subset of vectors and |
| their corresponding metadata is always fine. Consumption of vectors |
| should at least convert the unsupported input vectors to the |
| supported subset (for example Timestamp.millis to timestamp.micros |
| or int32 to int64).</p></li> |
| </ul> |
| </section> |
| <section id="extensibility"> |
| <h3>Extensibility<a class="headerlink" href="#extensibility" title="Permalink to this headline">#</a></h3> |
| <p>An execution engine implementor can also extend their memory |
| representation with their own vectors internally as long as they are |
| never exposed. Before sending data to another system expecting Arrow |
| data, these custom vectors should be converted to a type that exist in |
| the Arrow spec.</p> |
| </section> |
| </section> |
| </section> |
| |
| |
| </div> |
| |
| |
| <!-- Previous / next buttons --> |
| <div class='prev-next-area'> |
| <a class='left-prev' id="prev-link" href="Versioning.html" title="previous page"> |
| <i class="fas fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Format Versioning and Stability</p> |
| </div> |
| </a> |
| <a class='right-next' id="next-link" href="Flight.html" title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Arrow Flight RPC</p> |
| </div> |
| <i class="fas fa-angle-right"></i> |
| </a> |
| </div> |
| |
| </main> |
| |
| |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"></script> |
| <footer class="footer mt-5 mt-md-0"> |
| <div class="container"> |
| |
| <div class="footer-item"> |
| <p class="copyright"> |
| © Copyright 2016-2022 Apache Software Foundation.<br> |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| <p class="sphinx-version"> |
| Created using <a href="http://sphinx-doc.org/">Sphinx</a> 4.5.0.<br> |
| </p> |
| </div> |
| |
| </div> |
| </footer> |
| <script type="text/javascript" src="/docs/_static/versionwarning.js"></script> </body> |
| </html> |