| |
| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" /> |
| |
| <title>Data Types and In-Memory Data Model — Apache Arrow v17.0.0.dev52</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; |
| </script> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| |
| |
| <link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> |
| <script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script> |
| <script src="../_static/doctools.js"></script> |
| <script src="../_static/sphinx_highlight.js"></script> |
| <script src="../_static/clipboard.min.js"></script> |
| <script src="../_static/copybutton.js"></script> |
| <script src="../_static/design-tabs.js"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'python/data';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.15.2'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/python/data.html" /> |
| <link rel="icon" href="../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Compute Functions" href="compute.html" /> |
| <link rel="prev" title="Getting Started" href="getstarted.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i> |
| Back to top |
| </button> |
| |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__primary" |
| id="__primary"/> |
| <label class="overlay overlay-primary" for="__primary"></label> |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__secondary" |
| id="__secondary"/> |
| <label class="overlay overlay-secondary" for="__secondary"></label> |
| |
| <div class="search-button__wrapper"> |
| <div class="search-button__overlay"></div> |
| <div class="search-button__search-container"> |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| id="search-input" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form></div> |
| </div> |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar"> |
| <div class="bd-header__inner bd-page-width"> |
| <label class="sidebar-toggle primary-toggle" for="__primary"> |
| <span class="fa-solid fa-bars"></span> |
| </label> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev52 - Home"/> |
| <script>document.write(`<img src="../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev52 - Home"/>`);</script> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link dropdown-item nav-internal" href="index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| |
| <label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0"> |
| <span class="fa-solid fa-outdent"></span> |
| </label> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <div class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links-2" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link dropdown-item nav-internal" href="index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="install.html">Installing PyArrow</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="getstarted.html">Getting Started</a></li> |
| <li class="toctree-l1 current active"><a class="current reference internal" href="#">Data Types and In-Memory Data Model</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="compute.html">Compute Functions</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="memory.html">Memory and IO Interfaces</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="ipc.html">Streaming, Serialization, and IPC</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="filesystems.html">Filesystem Interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="numpy.html">NumPy Integration</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="pandas.html">Pandas Integration</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="interchange_protocol.html">Dataframe Interchange Protocol</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="dlpack.html">The DLPack Protocol</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="timestamps.html">Timestamps</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="orc.html">Reading and Writing the Apache ORC Format</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="csv.html">Reading and Writing CSV files</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="feather.html">Feather File Format</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="json.html">Reading JSON files</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="parquet.html">Reading and Writing the Apache Parquet Format</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="dataset.html">Tabular Datasets</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="extending_types.html">Extending pyarrow</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="integration.html">PyArrow Integrations</a><input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="integration/python_r.html">Integrating PyArrow with R</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="integration/python_java.html">Integrating PyArrow with Java</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="integration/extending.html">Using pyarrow from C++ and Cython Code</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="integration/cuda.html">CUDA Integration</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="env_vars.html">Environment Variables</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="api.html">API Reference</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/datatypes.html">Data Types and Schemas</a><input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-3"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.null.html">pyarrow.null</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.bool_.html">pyarrow.bool_</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.int8.html">pyarrow.int8</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.int16.html">pyarrow.int16</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.int32.html">pyarrow.int32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.int64.html">pyarrow.int64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.uint8.html">pyarrow.uint8</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.uint16.html">pyarrow.uint16</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.uint32.html">pyarrow.uint32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.uint64.html">pyarrow.uint64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.float16.html">pyarrow.float16</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.float32.html">pyarrow.float32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.float64.html">pyarrow.float64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.time32.html">pyarrow.time32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.time64.html">pyarrow.time64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.timestamp.html">pyarrow.timestamp</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.date32.html">pyarrow.date32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.date64.html">pyarrow.date64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.duration.html">pyarrow.duration</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.month_day_nano_interval.html">pyarrow.month_day_nano_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.binary.html">pyarrow.binary</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.string.html">pyarrow.string</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.utf8.html">pyarrow.utf8</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.large_binary.html">pyarrow.large_binary</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.large_string.html">pyarrow.large_string</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.large_utf8.html">pyarrow.large_utf8</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.binary_view.html">pyarrow.binary_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.string_view.html">pyarrow.string_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.decimal128.html">pyarrow.decimal128</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.list_.html">pyarrow.list_</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.large_list.html">pyarrow.large_list</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.list_view.html">pyarrow.list_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.large_list_view.html">pyarrow.large_list_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.map_.html">pyarrow.map_</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.struct.html">pyarrow.struct</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dictionary.html">pyarrow.dictionary</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.run_end_encoded.html">pyarrow.run_end_encoded</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.field.html">pyarrow.field</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.schema.html">pyarrow.schema</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.from_numpy_dtype.html">pyarrow.from_numpy_dtype</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.unify_schemas.html">pyarrow.unify_schemas</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DataType.html">pyarrow.DataType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DictionaryType.html">pyarrow.DictionaryType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ListType.html">pyarrow.ListType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.MapType.html">pyarrow.MapType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.StructType.html">pyarrow.StructType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UnionType.html">pyarrow.UnionType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.TimestampType.html">pyarrow.TimestampType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Time32Type.html">pyarrow.Time32Type</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Time64Type.html">pyarrow.Time64Type</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FixedSizeBinaryType.html">pyarrow.FixedSizeBinaryType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Decimal128Type.html">pyarrow.Decimal128Type</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Field.html">pyarrow.Field</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Schema.html">pyarrow.Schema</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.RunEndEncodedType.html">pyarrow.RunEndEncodedType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ExtensionType.html">pyarrow.ExtensionType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.PyExtensionType.html">pyarrow.PyExtensionType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.register_extension_type.html">pyarrow.register_extension_type</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.unregister_extension_type.html">pyarrow.unregister_extension_type</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_boolean.html">pyarrow.types.is_boolean</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_integer.html">pyarrow.types.is_integer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_signed_integer.html">pyarrow.types.is_signed_integer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_unsigned_integer.html">pyarrow.types.is_unsigned_integer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_int8.html">pyarrow.types.is_int8</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_int16.html">pyarrow.types.is_int16</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_int32.html">pyarrow.types.is_int32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_int64.html">pyarrow.types.is_int64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_uint8.html">pyarrow.types.is_uint8</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_uint16.html">pyarrow.types.is_uint16</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_uint32.html">pyarrow.types.is_uint32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_uint64.html">pyarrow.types.is_uint64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_floating.html">pyarrow.types.is_floating</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_float16.html">pyarrow.types.is_float16</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_float32.html">pyarrow.types.is_float32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_float64.html">pyarrow.types.is_float64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_decimal.html">pyarrow.types.is_decimal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_decimal128.html">pyarrow.types.is_decimal128</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_decimal256.html">pyarrow.types.is_decimal256</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_list.html">pyarrow.types.is_list</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_large_list.html">pyarrow.types.is_large_list</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_fixed_size_list.html">pyarrow.types.is_fixed_size_list</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_list_view.html">pyarrow.types.is_list_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_large_list_view.html">pyarrow.types.is_large_list_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_struct.html">pyarrow.types.is_struct</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_union.html">pyarrow.types.is_union</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_nested.html">pyarrow.types.is_nested</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_run_end_encoded.html">pyarrow.types.is_run_end_encoded</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_temporal.html">pyarrow.types.is_temporal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_timestamp.html">pyarrow.types.is_timestamp</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_date.html">pyarrow.types.is_date</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_date32.html">pyarrow.types.is_date32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_date64.html">pyarrow.types.is_date64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_time.html">pyarrow.types.is_time</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_time32.html">pyarrow.types.is_time32</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_time64.html">pyarrow.types.is_time64</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_duration.html">pyarrow.types.is_duration</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_interval.html">pyarrow.types.is_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_null.html">pyarrow.types.is_null</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_binary.html">pyarrow.types.is_binary</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_unicode.html">pyarrow.types.is_unicode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_string.html">pyarrow.types.is_string</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_large_binary.html">pyarrow.types.is_large_binary</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_large_unicode.html">pyarrow.types.is_large_unicode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_large_string.html">pyarrow.types.is_large_string</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_binary_view.html">pyarrow.types.is_binary_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_string_view.html">pyarrow.types.is_string_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_fixed_size_binary.html">pyarrow.types.is_fixed_size_binary</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_map.html">pyarrow.types.is_map</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_dictionary.html">pyarrow.types.is_dictionary</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.types.is_primitive.html">pyarrow.types.is_primitive</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/arrays.html">Arrays and Scalars</a><input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-4"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.array.html">pyarrow.array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.nulls.html">pyarrow.nulls</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Array.html">pyarrow.Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.BooleanArray.html">pyarrow.BooleanArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FloatingPointArray.html">pyarrow.FloatingPointArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.IntegerArray.html">pyarrow.IntegerArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int8Array.html">pyarrow.Int8Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int16Array.html">pyarrow.Int16Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int32Array.html">pyarrow.Int32Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int64Array.html">pyarrow.Int64Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.NullArray.html">pyarrow.NullArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.NumericArray.html">pyarrow.NumericArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt8Array.html">pyarrow.UInt8Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt16Array.html">pyarrow.UInt16Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt32Array.html">pyarrow.UInt32Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt64Array.html">pyarrow.UInt64Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.HalfFloatArray.html">pyarrow.HalfFloatArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FloatArray.html">pyarrow.FloatArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DoubleArray.html">pyarrow.DoubleArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.BinaryArray.html">pyarrow.BinaryArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.StringArray.html">pyarrow.StringArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FixedSizeBinaryArray.html">pyarrow.FixedSizeBinaryArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeBinaryArray.html">pyarrow.LargeBinaryArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeStringArray.html">pyarrow.LargeStringArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Time32Array.html">pyarrow.Time32Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Time64Array.html">pyarrow.Time64Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Date32Array.html">pyarrow.Date32Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Date64Array.html">pyarrow.Date64Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.TimestampArray.html">pyarrow.TimestampArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DurationArray.html">pyarrow.DurationArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.MonthDayNanoIntervalArray.html">pyarrow.MonthDayNanoIntervalArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Decimal128Array.html">pyarrow.Decimal128Array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DictionaryArray.html">pyarrow.DictionaryArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ListArray.html">pyarrow.ListArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FixedSizeListArray.html">pyarrow.FixedSizeListArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeListArray.html">pyarrow.LargeListArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ListViewArray.html">pyarrow.ListViewArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeListViewArray.html">pyarrow.LargeListViewArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.MapArray.html">pyarrow.MapArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.RunEndEncodedArray.html">pyarrow.RunEndEncodedArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.StructArray.html">pyarrow.StructArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UnionArray.html">pyarrow.UnionArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ExtensionArray.html">pyarrow.ExtensionArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FixedShapeTensorArray.html">pyarrow.FixedShapeTensorArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.scalar.html">pyarrow.scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.NA.html">pyarrow.NA</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Scalar.html">pyarrow.Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.BooleanScalar.html">pyarrow.BooleanScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int8Scalar.html">pyarrow.Int8Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int16Scalar.html">pyarrow.Int16Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int32Scalar.html">pyarrow.Int32Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Int64Scalar.html">pyarrow.Int64Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt8Scalar.html">pyarrow.UInt8Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt16Scalar.html">pyarrow.UInt16Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt32Scalar.html">pyarrow.UInt32Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UInt64Scalar.html">pyarrow.UInt64Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.HalfFloatScalar.html">pyarrow.HalfFloatScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FloatScalar.html">pyarrow.FloatScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DoubleScalar.html">pyarrow.DoubleScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.BinaryScalar.html">pyarrow.BinaryScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.StringScalar.html">pyarrow.StringScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FixedSizeBinaryScalar.html">pyarrow.FixedSizeBinaryScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeBinaryScalar.html">pyarrow.LargeBinaryScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeStringScalar.html">pyarrow.LargeStringScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.BinaryViewScalar.html">pyarrow.BinaryViewScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.StringViewScalar.html">pyarrow.StringViewScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Time32Scalar.html">pyarrow.Time32Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Time64Scalar.html">pyarrow.Time64Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Date32Scalar.html">pyarrow.Date32Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Date64Scalar.html">pyarrow.Date64Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.TimestampScalar.html">pyarrow.TimestampScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DurationScalar.html">pyarrow.DurationScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.MonthDayNanoIntervalScalar.html">pyarrow.MonthDayNanoIntervalScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Decimal128Scalar.html">pyarrow.Decimal128Scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.DictionaryScalar.html">pyarrow.DictionaryScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.RunEndEncodedScalar.html">pyarrow.RunEndEncodedScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ListScalar.html">pyarrow.ListScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeListScalar.html">pyarrow.LargeListScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ListViewScalar.html">pyarrow.ListViewScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.LargeListViewScalar.html">pyarrow.LargeListViewScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.MapScalar.html">pyarrow.MapScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.StructScalar.html">pyarrow.StructScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.UnionScalar.html">pyarrow.UnionScalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ExtensionScalar.html">pyarrow.ExtensionScalar</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/memory.html">Buffers and Memory</a><input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-5"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.allocate_buffer.html">pyarrow.allocate_buffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.py_buffer.html">pyarrow.py_buffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.foreign_buffer.html">pyarrow.foreign_buffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Buffer.html">pyarrow.Buffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ResizableBuffer.html">pyarrow.ResizableBuffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Codec.html">pyarrow.Codec</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compress.html">pyarrow.compress</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.decompress.html">pyarrow.decompress</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.MemoryPool.html">pyarrow.MemoryPool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.default_memory_pool.html">pyarrow.default_memory_pool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.jemalloc_memory_pool.html">pyarrow.jemalloc_memory_pool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.mimalloc_memory_pool.html">pyarrow.mimalloc_memory_pool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.system_memory_pool.html">pyarrow.system_memory_pool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.jemalloc_set_decay_ms.html">pyarrow.jemalloc_set_decay_ms</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.set_memory_pool.html">pyarrow.set_memory_pool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.log_memory_allocations.html">pyarrow.log_memory_allocations</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.total_allocated_bytes.html">pyarrow.total_allocated_bytes</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/tables.html">Tables and Tensors</a><input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-6"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.chunked_array.html">pyarrow.chunked_array</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.concat_arrays.html">pyarrow.concat_arrays</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.concat_tables.html">pyarrow.concat_tables</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.record_batch.html">pyarrow.record_batch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.table.html">pyarrow.table</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ChunkedArray.html">pyarrow.ChunkedArray</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.RecordBatch.html">pyarrow.RecordBatch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Table.html">pyarrow.Table</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.TableGroupBy.html">pyarrow.TableGroupBy</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.RecordBatchReader.html">pyarrow.RecordBatchReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.interchange.from_dataframe.html">pyarrow.interchange.from_dataframe</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.Tensor.html">pyarrow.Tensor</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/compute.html">Compute Functions</a><input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-7"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.all.html">pyarrow.compute.all</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.any.html">pyarrow.compute.any</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.approximate_median.html">pyarrow.compute.approximate_median</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.count.html">pyarrow.compute.count</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.count_distinct.html">pyarrow.compute.count_distinct</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.index.html">pyarrow.compute.index</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.max.html">pyarrow.compute.max</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.mean.html">pyarrow.compute.mean</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.min.html">pyarrow.compute.min</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.min_max.html">pyarrow.compute.min_max</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.mode.html">pyarrow.compute.mode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.product.html">pyarrow.compute.product</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.quantile.html">pyarrow.compute.quantile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.stddev.html">pyarrow.compute.stddev</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.sum.html">pyarrow.compute.sum</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.tdigest.html">pyarrow.compute.tdigest</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.variance.html">pyarrow.compute.variance</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cumulative_sum.html">pyarrow.compute.cumulative_sum</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cumulative_sum_checked.html">pyarrow.compute.cumulative_sum_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cumulative_prod.html">pyarrow.compute.cumulative_prod</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cumulative_prod_checked.html">pyarrow.compute.cumulative_prod_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cumulative_max.html">pyarrow.compute.cumulative_max</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cumulative_min.html">pyarrow.compute.cumulative_min</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.abs.html">pyarrow.compute.abs</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.abs_checked.html">pyarrow.compute.abs_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.add.html">pyarrow.compute.add</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.add_checked.html">pyarrow.compute.add_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.divide.html">pyarrow.compute.divide</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.divide_checked.html">pyarrow.compute.divide_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.multiply.html">pyarrow.compute.multiply</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.multiply_checked.html">pyarrow.compute.multiply_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.negate.html">pyarrow.compute.negate</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.negate_checked.html">pyarrow.compute.negate_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.power.html">pyarrow.compute.power</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.power_checked.html">pyarrow.compute.power_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.sign.html">pyarrow.compute.sign</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.sqrt.html">pyarrow.compute.sqrt</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.sqrt_checked.html">pyarrow.compute.sqrt_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.subtract.html">pyarrow.compute.subtract</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.subtract_checked.html">pyarrow.compute.subtract_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_and.html">pyarrow.compute.bit_wise_and</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_not.html">pyarrow.compute.bit_wise_not</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_or.html">pyarrow.compute.bit_wise_or</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_xor.html">pyarrow.compute.bit_wise_xor</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.shift_left.html">pyarrow.compute.shift_left</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.shift_left_checked.html">pyarrow.compute.shift_left_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.shift_right.html">pyarrow.compute.shift_right</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.shift_right_checked.html">pyarrow.compute.shift_right_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ceil.html">pyarrow.compute.ceil</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.floor.html">pyarrow.compute.floor</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.round.html">pyarrow.compute.round</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.round_to_multiple.html">pyarrow.compute.round_to_multiple</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.trunc.html">pyarrow.compute.trunc</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ln.html">pyarrow.compute.ln</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ln_checked.html">pyarrow.compute.ln_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.log10.html">pyarrow.compute.log10</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.log10_checked.html">pyarrow.compute.log10_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.log1p.html">pyarrow.compute.log1p</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.log1p_checked.html">pyarrow.compute.log1p_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.log2.html">pyarrow.compute.log2</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.log2_checked.html">pyarrow.compute.log2_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.logb.html">pyarrow.compute.logb</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.logb_checked.html">pyarrow.compute.logb_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.acos.html">pyarrow.compute.acos</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.acos_checked.html">pyarrow.compute.acos_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.asin.html">pyarrow.compute.asin</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.asin_checked.html">pyarrow.compute.asin_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.atan.html">pyarrow.compute.atan</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.atan2.html">pyarrow.compute.atan2</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cos.html">pyarrow.compute.cos</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cos_checked.html">pyarrow.compute.cos_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.sin.html">pyarrow.compute.sin</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.sin_checked.html">pyarrow.compute.sin_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.tan.html">pyarrow.compute.tan</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.tan_checked.html">pyarrow.compute.tan_checked</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.equal.html">pyarrow.compute.equal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.greater.html">pyarrow.compute.greater</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.greater_equal.html">pyarrow.compute.greater_equal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.less.html">pyarrow.compute.less</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.less_equal.html">pyarrow.compute.less_equal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.not_equal.html">pyarrow.compute.not_equal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.max_element_wise.html">pyarrow.compute.max_element_wise</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.min_element_wise.html">pyarrow.compute.min_element_wise</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.and_.html">pyarrow.compute.and_</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.and_kleene.html">pyarrow.compute.and_kleene</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.and_not.html">pyarrow.compute.and_not</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.and_not_kleene.html">pyarrow.compute.and_not_kleene</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.invert.html">pyarrow.compute.invert</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.or_.html">pyarrow.compute.or_</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.or_kleene.html">pyarrow.compute.or_kleene</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.xor.html">pyarrow.compute.xor</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_alnum.html">pyarrow.compute.ascii_is_alnum</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_alpha.html">pyarrow.compute.ascii_is_alpha</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_decimal.html">pyarrow.compute.ascii_is_decimal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_lower.html">pyarrow.compute.ascii_is_lower</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_printable.html">pyarrow.compute.ascii_is_printable</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_space.html">pyarrow.compute.ascii_is_space</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_upper.html">pyarrow.compute.ascii_is_upper</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_alnum.html">pyarrow.compute.utf8_is_alnum</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_alpha.html">pyarrow.compute.utf8_is_alpha</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_decimal.html">pyarrow.compute.utf8_is_decimal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_digit.html">pyarrow.compute.utf8_is_digit</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_lower.html">pyarrow.compute.utf8_is_lower</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_numeric.html">pyarrow.compute.utf8_is_numeric</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_printable.html">pyarrow.compute.utf8_is_printable</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_space.html">pyarrow.compute.utf8_is_space</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_upper.html">pyarrow.compute.utf8_is_upper</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_title.html">pyarrow.compute.ascii_is_title</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_title.html">pyarrow.compute.utf8_is_title</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.string_is_ascii.html">pyarrow.compute.string_is_ascii</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_capitalize.html">pyarrow.compute.ascii_capitalize</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_lower.html">pyarrow.compute.ascii_lower</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_reverse.html">pyarrow.compute.ascii_reverse</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_swapcase.html">pyarrow.compute.ascii_swapcase</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_title.html">pyarrow.compute.ascii_title</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_upper.html">pyarrow.compute.ascii_upper</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.binary_length.html">pyarrow.compute.binary_length</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.binary_repeat.html">pyarrow.compute.binary_repeat</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.binary_replace_slice.html">pyarrow.compute.binary_replace_slice</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.binary_reverse.html">pyarrow.compute.binary_reverse</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.replace_substring.html">pyarrow.compute.replace_substring</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.replace_substring_regex.html">pyarrow.compute.replace_substring_regex</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_capitalize.html">pyarrow.compute.utf8_capitalize</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_length.html">pyarrow.compute.utf8_length</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_lower.html">pyarrow.compute.utf8_lower</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_replace_slice.html">pyarrow.compute.utf8_replace_slice</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_reverse.html">pyarrow.compute.utf8_reverse</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_swapcase.html">pyarrow.compute.utf8_swapcase</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_title.html">pyarrow.compute.utf8_title</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_upper.html">pyarrow.compute.utf8_upper</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_center.html">pyarrow.compute.ascii_center</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_lpad.html">pyarrow.compute.ascii_lpad</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_rpad.html">pyarrow.compute.ascii_rpad</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_center.html">pyarrow.compute.utf8_center</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_lpad.html">pyarrow.compute.utf8_lpad</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_rpad.html">pyarrow.compute.utf8_rpad</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_ltrim.html">pyarrow.compute.ascii_ltrim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_ltrim_whitespace.html">pyarrow.compute.ascii_ltrim_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_rtrim.html">pyarrow.compute.ascii_rtrim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_rtrim_whitespace.html">pyarrow.compute.ascii_rtrim_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_trim.html">pyarrow.compute.ascii_trim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_trim_whitespace.html">pyarrow.compute.ascii_trim_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_ltrim.html">pyarrow.compute.utf8_ltrim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_ltrim_whitespace.html">pyarrow.compute.utf8_ltrim_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_rtrim.html">pyarrow.compute.utf8_rtrim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_rtrim_whitespace.html">pyarrow.compute.utf8_rtrim_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_trim.html">pyarrow.compute.utf8_trim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_trim_whitespace.html">pyarrow.compute.utf8_trim_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ascii_split_whitespace.html">pyarrow.compute.ascii_split_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.split_pattern.html">pyarrow.compute.split_pattern</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.split_pattern_regex.html">pyarrow.compute.split_pattern_regex</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_split_whitespace.html">pyarrow.compute.utf8_split_whitespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.extract_regex.html">pyarrow.compute.extract_regex</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.binary_join.html">pyarrow.compute.binary_join</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.binary_join_element_wise.html">pyarrow.compute.binary_join_element_wise</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.binary_slice.html">pyarrow.compute.binary_slice</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.utf8_slice_codeunits.html">pyarrow.compute.utf8_slice_codeunits</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.count_substring.html">pyarrow.compute.count_substring</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.count_substring_regex.html">pyarrow.compute.count_substring_regex</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ends_with.html">pyarrow.compute.ends_with</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.find_substring.html">pyarrow.compute.find_substring</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.find_substring_regex.html">pyarrow.compute.find_substring_regex</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.index_in.html">pyarrow.compute.index_in</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.is_in.html">pyarrow.compute.is_in</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.match_like.html">pyarrow.compute.match_like</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.match_substring.html">pyarrow.compute.match_substring</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.match_substring_regex.html">pyarrow.compute.match_substring_regex</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.starts_with.html">pyarrow.compute.starts_with</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.indices_nonzero.html">pyarrow.compute.indices_nonzero</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.is_finite.html">pyarrow.compute.is_finite</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.is_inf.html">pyarrow.compute.is_inf</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.is_nan.html">pyarrow.compute.is_nan</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.is_null.html">pyarrow.compute.is_null</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.is_valid.html">pyarrow.compute.is_valid</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.true_unless_null.html">pyarrow.compute.true_unless_null</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.case_when.html">pyarrow.compute.case_when</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.choose.html">pyarrow.compute.choose</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.coalesce.html">pyarrow.compute.coalesce</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.if_else.html">pyarrow.compute.if_else</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.cast.html">pyarrow.compute.cast</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ceil_temporal.html">pyarrow.compute.ceil_temporal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.floor_temporal.html">pyarrow.compute.floor_temporal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.round_temporal.html">pyarrow.compute.round_temporal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.run_end_decode.html">pyarrow.compute.run_end_decode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.run_end_encode.html">pyarrow.compute.run_end_encode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.strftime.html">pyarrow.compute.strftime</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.strptime.html">pyarrow.compute.strptime</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.day.html">pyarrow.compute.day</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.day_of_week.html">pyarrow.compute.day_of_week</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.day_of_year.html">pyarrow.compute.day_of_year</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.hour.html">pyarrow.compute.hour</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.iso_week.html">pyarrow.compute.iso_week</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.iso_year.html">pyarrow.compute.iso_year</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.iso_calendar.html">pyarrow.compute.iso_calendar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.is_leap_year.html">pyarrow.compute.is_leap_year</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.microsecond.html">pyarrow.compute.microsecond</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.millisecond.html">pyarrow.compute.millisecond</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.minute.html">pyarrow.compute.minute</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.month.html">pyarrow.compute.month</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.nanosecond.html">pyarrow.compute.nanosecond</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.quarter.html">pyarrow.compute.quarter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.second.html">pyarrow.compute.second</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.subsecond.html">pyarrow.compute.subsecond</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.us_week.html">pyarrow.compute.us_week</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.us_year.html">pyarrow.compute.us_year</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.week.html">pyarrow.compute.week</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.year.html">pyarrow.compute.year</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.year_month_day.html">pyarrow.compute.year_month_day</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.day_time_interval_between.html">pyarrow.compute.day_time_interval_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.days_between.html">pyarrow.compute.days_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.hours_between.html">pyarrow.compute.hours_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.microseconds_between.html">pyarrow.compute.microseconds_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.milliseconds_between.html">pyarrow.compute.milliseconds_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.minutes_between.html">pyarrow.compute.minutes_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.month_day_nano_interval_between.html">pyarrow.compute.month_day_nano_interval_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.month_interval_between.html">pyarrow.compute.month_interval_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.nanoseconds_between.html">pyarrow.compute.nanoseconds_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.quarters_between.html">pyarrow.compute.quarters_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.seconds_between.html">pyarrow.compute.seconds_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.weeks_between.html">pyarrow.compute.weeks_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.years_between.html">pyarrow.compute.years_between</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.assume_timezone.html">pyarrow.compute.assume_timezone</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.local_timestamp.html">pyarrow.compute.local_timestamp</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.dictionary_encode.html">pyarrow.compute.dictionary_encode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.unique.html">pyarrow.compute.unique</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.value_counts.html">pyarrow.compute.value_counts</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.array_filter.html">pyarrow.compute.array_filter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.array_take.html">pyarrow.compute.array_take</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.drop_null.html">pyarrow.compute.drop_null</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.filter.html">pyarrow.compute.filter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.take.html">pyarrow.compute.take</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.array_sort_indices.html">pyarrow.compute.array_sort_indices</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.partition_nth_indices.html">pyarrow.compute.partition_nth_indices</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.select_k_unstable.html">pyarrow.compute.select_k_unstable</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.sort_indices.html">pyarrow.compute.sort_indices</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.fill_null.html">pyarrow.compute.fill_null</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.fill_null_backward.html">pyarrow.compute.fill_null_backward</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.fill_null_forward.html">pyarrow.compute.fill_null_forward</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.list_element.html">pyarrow.compute.list_element</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.list_flatten.html">pyarrow.compute.list_flatten</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.list_parent_indices.html">pyarrow.compute.list_parent_indices</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.list_slice.html">pyarrow.compute.list_slice</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.list_value_length.html">pyarrow.compute.list_value_length</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.make_struct.html">pyarrow.compute.make_struct</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.map_lookup.html">pyarrow.compute.map_lookup</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.replace_with_mask.html">pyarrow.compute.replace_with_mask</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.struct_field.html">pyarrow.compute.struct_field</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.pairwise_diff.html">pyarrow.compute.pairwise_diff</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ArraySortOptions.html">pyarrow.compute.ArraySortOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.AssumeTimezoneOptions.html">pyarrow.compute.AssumeTimezoneOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.CastOptions.html">pyarrow.compute.CastOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.CountOptions.html">pyarrow.compute.CountOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.CountOptions.html">pyarrow.compute.CountOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.CumulativeSumOptions.html">pyarrow.compute.CumulativeSumOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.DayOfWeekOptions.html">pyarrow.compute.DayOfWeekOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.DictionaryEncodeOptions.html">pyarrow.compute.DictionaryEncodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ElementWiseAggregateOptions.html">pyarrow.compute.ElementWiseAggregateOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ExtractRegexOptions.html">pyarrow.compute.ExtractRegexOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.FilterOptions.html">pyarrow.compute.FilterOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.IndexOptions.html">pyarrow.compute.IndexOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.JoinOptions.html">pyarrow.compute.JoinOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ListSliceOptions.html">pyarrow.compute.ListSliceOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.MakeStructOptions.html">pyarrow.compute.MakeStructOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.MapLookupOptions.html">pyarrow.compute.MapLookupOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.MatchSubstringOptions.html">pyarrow.compute.MatchSubstringOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ModeOptions.html">pyarrow.compute.ModeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.NullOptions.html">pyarrow.compute.NullOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.PadOptions.html">pyarrow.compute.PadOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.PairwiseOptions.html">pyarrow.compute.PairwiseOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.PartitionNthOptions.html">pyarrow.compute.PartitionNthOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.QuantileOptions.html">pyarrow.compute.QuantileOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ReplaceSliceOptions.html">pyarrow.compute.ReplaceSliceOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ReplaceSubstringOptions.html">pyarrow.compute.ReplaceSubstringOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.RoundOptions.html">pyarrow.compute.RoundOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.RoundTemporalOptions.html">pyarrow.compute.RoundTemporalOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.RoundToMultipleOptions.html">pyarrow.compute.RoundToMultipleOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.RunEndEncodeOptions.html">pyarrow.compute.RunEndEncodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ScalarAggregateOptions.html">pyarrow.compute.ScalarAggregateOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.ScalarAggregateOptions.html">pyarrow.compute.ScalarAggregateOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.SelectKOptions.html">pyarrow.compute.SelectKOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.SetLookupOptions.html">pyarrow.compute.SetLookupOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.SliceOptions.html">pyarrow.compute.SliceOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.SortOptions.html">pyarrow.compute.SortOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.SplitOptions.html">pyarrow.compute.SplitOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.SplitPatternOptions.html">pyarrow.compute.SplitPatternOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.StrftimeOptions.html">pyarrow.compute.StrftimeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.StrptimeOptions.html">pyarrow.compute.StrptimeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.StructFieldOptions.html">pyarrow.compute.StructFieldOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.TakeOptions.html">pyarrow.compute.TakeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.TDigestOptions.html">pyarrow.compute.TDigestOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.TDigestOptions.html">pyarrow.compute.TDigestOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.TrimOptions.html">pyarrow.compute.TrimOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.VarianceOptions.html">pyarrow.compute.VarianceOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.WeekOptions.html">pyarrow.compute.WeekOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.register_scalar_function.html">pyarrow.compute.register_scalar_function</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.compute.UdfContext.html">pyarrow.compute.UdfContext</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/acero.html">Acero - Streaming Execution Engine</a><input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-8"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.Declaration.html">pyarrow.acero.Declaration</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.ExecNodeOptions.html">pyarrow.acero.ExecNodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.TableSourceNodeOptions.html">pyarrow.acero.TableSourceNodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.ScanNodeOptions.html">pyarrow.acero.ScanNodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.FilterNodeOptions.html">pyarrow.acero.FilterNodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.ProjectNodeOptions.html">pyarrow.acero.ProjectNodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.AggregateNodeOptions.html">pyarrow.acero.AggregateNodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.OrderByNodeOptions.html">pyarrow.acero.OrderByNodeOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.acero.HashJoinNodeOptions.html">pyarrow.acero.HashJoinNodeOptions</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/substrait.html">Substrait</a><input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-9"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.substrait.run_query.html">pyarrow.substrait.run_query</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.substrait.BoundExpressions.html">pyarrow.substrait.BoundExpressions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.substrait.deserialize_expressions.html">pyarrow.substrait.deserialize_expressions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.substrait.serialize_expressions.html">pyarrow.substrait.serialize_expressions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.substrait.get_supported_functions.html">pyarrow.substrait.get_supported_functions</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/files.html">Streams and File Access</a><input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-10"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.input_stream.html">pyarrow.input_stream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.output_stream.html">pyarrow.output_stream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.memory_map.html">pyarrow.memory_map</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.create_memory_map.html">pyarrow.create_memory_map</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.NativeFile.html">pyarrow.NativeFile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.OSFile.html">pyarrow.OSFile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.PythonFile.html">pyarrow.PythonFile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.BufferReader.html">pyarrow.BufferReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.BufferOutputStream.html">pyarrow.BufferOutputStream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.FixedSizeBufferWriter.html">pyarrow.FixedSizeBufferWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.MemoryMappedFile.html">pyarrow.MemoryMappedFile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.CompressedInputStream.html">pyarrow.CompressedInputStream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.CompressedOutputStream.html">pyarrow.CompressedOutputStream</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/ipc.html">Serialization and IPC</a><input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-11"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.new_file.html">pyarrow.ipc.new_file</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.open_file.html">pyarrow.ipc.open_file</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.new_stream.html">pyarrow.ipc.new_stream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.open_stream.html">pyarrow.ipc.open_stream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.read_message.html">pyarrow.ipc.read_message</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.read_record_batch.html">pyarrow.ipc.read_record_batch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.get_record_batch_size.html">pyarrow.ipc.get_record_batch_size</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.read_tensor.html">pyarrow.ipc.read_tensor</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.write_tensor.html">pyarrow.ipc.write_tensor</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.get_tensor_size.html">pyarrow.ipc.get_tensor_size</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.IpcReadOptions.html">pyarrow.ipc.IpcReadOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.IpcWriteOptions.html">pyarrow.ipc.IpcWriteOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.Message.html">pyarrow.ipc.Message</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.MessageReader.html">pyarrow.ipc.MessageReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchFileReader.html">pyarrow.ipc.RecordBatchFileReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchFileWriter.html">pyarrow.ipc.RecordBatchFileWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchStreamReader.html">pyarrow.ipc.RecordBatchStreamReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchStreamWriter.html">pyarrow.ipc.RecordBatchStreamWriter</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/flight.html">Arrow Flight</a><input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-12"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.Action.html">pyarrow.flight.Action</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ActionType.html">pyarrow.flight.ActionType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.DescriptorType.html">pyarrow.flight.DescriptorType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightDescriptor.html">pyarrow.flight.FlightDescriptor</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightEndpoint.html">pyarrow.flight.FlightEndpoint</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightInfo.html">pyarrow.flight.FlightInfo</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.Location.html">pyarrow.flight.Location</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.MetadataRecordBatchReader.html">pyarrow.flight.MetadataRecordBatchReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.MetadataRecordBatchWriter.html">pyarrow.flight.MetadataRecordBatchWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.Ticket.html">pyarrow.flight.Ticket</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.Result.html">pyarrow.flight.Result</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.connect.html">pyarrow.flight.connect</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightCallOptions.html">pyarrow.flight.FlightCallOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightClient.html">pyarrow.flight.FlightClient</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightStreamReader.html">pyarrow.flight.FlightStreamReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightStreamWriter.html">pyarrow.flight.FlightStreamWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ClientMiddlewareFactory.html">pyarrow.flight.ClientMiddlewareFactory</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ClientMiddleware.html">pyarrow.flight.ClientMiddleware</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightDataStream.html">pyarrow.flight.FlightDataStream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightMetadataWriter.html">pyarrow.flight.FlightMetadataWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightServerBase.html">pyarrow.flight.FlightServerBase</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.GeneratorStream.html">pyarrow.flight.GeneratorStream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.RecordBatchStream.html">pyarrow.flight.RecordBatchStream</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ServerCallContext.html">pyarrow.flight.ServerCallContext</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ServerMiddlewareFactory.html">pyarrow.flight.ServerMiddlewareFactory</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ServerMiddleware.html">pyarrow.flight.ServerMiddleware</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ClientAuthHandler.html">pyarrow.flight.ClientAuthHandler</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.ServerAuthHandler.html">pyarrow.flight.ServerAuthHandler</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightError.html">pyarrow.flight.FlightError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightCancelledError.html">pyarrow.flight.FlightCancelledError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightInternalError.html">pyarrow.flight.FlightInternalError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightServerError.html">pyarrow.flight.FlightServerError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightTimedOutError.html">pyarrow.flight.FlightTimedOutError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightUnauthenticatedError.html">pyarrow.flight.FlightUnauthenticatedError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightUnauthorizedError.html">pyarrow.flight.FlightUnauthorizedError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightUnavailableError.html">pyarrow.flight.FlightUnavailableError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightWriteSizeExceededError.html">pyarrow.flight.FlightWriteSizeExceededError</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.FlightMethod.html">pyarrow.flight.FlightMethod</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.flight.CallInfo.html">pyarrow.flight.CallInfo</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/formats.html">Tabular File Formats</a><input class="toctree-checkbox" id="toctree-checkbox-13" name="toctree-checkbox-13" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-13"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.ConvertOptions.html">pyarrow.csv.ConvertOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.CSVStreamingReader.html">pyarrow.csv.CSVStreamingReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.CSVWriter.html">pyarrow.csv.CSVWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.ISO8601.html">pyarrow.csv.ISO8601</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.ParseOptions.html">pyarrow.csv.ParseOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.ReadOptions.html">pyarrow.csv.ReadOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.WriteOptions.html">pyarrow.csv.WriteOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.open_csv.html">pyarrow.csv.open_csv</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.read_csv.html">pyarrow.csv.read_csv</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.write_csv.html">pyarrow.csv.write_csv</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.csv.InvalidRow.html">pyarrow.csv.InvalidRow</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.feather.read_feather.html">pyarrow.feather.read_feather</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.feather.read_table.html">pyarrow.feather.read_table</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.feather.write_feather.html">pyarrow.feather.write_feather</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.json.ReadOptions.html">pyarrow.json.ReadOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.json.ParseOptions.html">pyarrow.json.ParseOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.json.read_json.html">pyarrow.json.read_json</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.ParquetDataset.html">pyarrow.parquet.ParquetDataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.ParquetFile.html">pyarrow.parquet.ParquetFile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.ParquetWriter.html">pyarrow.parquet.ParquetWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.read_table.html">pyarrow.parquet.read_table</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.read_metadata.html">pyarrow.parquet.read_metadata</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.read_pandas.html">pyarrow.parquet.read_pandas</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.read_schema.html">pyarrow.parquet.read_schema</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.write_metadata.html">pyarrow.parquet.write_metadata</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.write_table.html">pyarrow.parquet.write_table</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.write_to_dataset.html">pyarrow.parquet.write_to_dataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.FileMetaData.html">pyarrow.parquet.FileMetaData</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.RowGroupMetaData.html">pyarrow.parquet.RowGroupMetaData</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.SortingColumn.html">pyarrow.parquet.SortingColumn</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.ColumnChunkMetaData.html">pyarrow.parquet.ColumnChunkMetaData</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.Statistics.html">pyarrow.parquet.Statistics</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.ParquetSchema.html">pyarrow.parquet.ParquetSchema</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.ColumnSchema.html">pyarrow.parquet.ColumnSchema</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.ParquetLogicalType.html">pyarrow.parquet.ParquetLogicalType</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.encryption.CryptoFactory.html">pyarrow.parquet.encryption.CryptoFactory</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.encryption.KmsClient.html">pyarrow.parquet.encryption.KmsClient</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.encryption.KmsConnectionConfig.html">pyarrow.parquet.encryption.KmsConnectionConfig</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.encryption.EncryptionConfiguration.html">pyarrow.parquet.encryption.EncryptionConfiguration</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.parquet.encryption.DecryptionConfiguration.html">pyarrow.parquet.encryption.DecryptionConfiguration</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.orc.ORCFile.html">pyarrow.orc.ORCFile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.orc.ORCWriter.html">pyarrow.orc.ORCWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.orc.read_table.html">pyarrow.orc.read_table</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.orc.write_table.html">pyarrow.orc.write_table</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/filesystems.html">Filesystems</a><input class="toctree-checkbox" id="toctree-checkbox-14" name="toctree-checkbox-14" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-14"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.FileInfo.html">pyarrow.fs.FileInfo</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.FileSelector.html">pyarrow.fs.FileSelector</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.FileSystem.html">pyarrow.fs.FileSystem</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.LocalFileSystem.html">pyarrow.fs.LocalFileSystem</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.S3FileSystem.html">pyarrow.fs.S3FileSystem</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.GcsFileSystem.html">pyarrow.fs.GcsFileSystem</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.HadoopFileSystem.html">pyarrow.fs.HadoopFileSystem</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.SubTreeFileSystem.html">pyarrow.fs.SubTreeFileSystem</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.PyFileSystem.html">pyarrow.fs.PyFileSystem</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.FileSystemHandler.html">pyarrow.fs.FileSystemHandler</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.FSSpecHandler.html">pyarrow.fs.FSSpecHandler</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.copy_files.html">pyarrow.fs.copy_files</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.initialize_s3.html">pyarrow.fs.initialize_s3</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.finalize_s3.html">pyarrow.fs.finalize_s3</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.resolve_s3_region.html">pyarrow.fs.resolve_s3_region</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.fs.S3LogLevel.html">pyarrow.fs.S3LogLevel</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/dataset.html">Dataset</a><input class="toctree-checkbox" id="toctree-checkbox-15" name="toctree-checkbox-15" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-15"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.dataset.html">pyarrow.dataset.dataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.parquet_dataset.html">pyarrow.dataset.parquet_dataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.partitioning.html">pyarrow.dataset.partitioning</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.field.html">pyarrow.dataset.field</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.scalar.html">pyarrow.dataset.scalar</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.write_dataset.html">pyarrow.dataset.write_dataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.FileFormat.html">pyarrow.dataset.FileFormat</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.CsvFileFormat.html">pyarrow.dataset.CsvFileFormat</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.CsvFragmentScanOptions.html">pyarrow.dataset.CsvFragmentScanOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.IpcFileFormat.html">pyarrow.dataset.IpcFileFormat</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.JsonFileFormat.html">pyarrow.dataset.JsonFileFormat</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.ParquetFileFormat.html">pyarrow.dataset.ParquetFileFormat</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.ParquetReadOptions.html">pyarrow.dataset.ParquetReadOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.ParquetFragmentScanOptions.html">pyarrow.dataset.ParquetFragmentScanOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.ParquetFileFragment.html">pyarrow.dataset.ParquetFileFragment</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.OrcFileFormat.html">pyarrow.dataset.OrcFileFormat</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.Partitioning.html">pyarrow.dataset.Partitioning</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.PartitioningFactory.html">pyarrow.dataset.PartitioningFactory</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.DirectoryPartitioning.html">pyarrow.dataset.DirectoryPartitioning</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.HivePartitioning.html">pyarrow.dataset.HivePartitioning</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.FilenamePartitioning.html">pyarrow.dataset.FilenamePartitioning</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.Dataset.html">pyarrow.dataset.Dataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.FileSystemDataset.html">pyarrow.dataset.FileSystemDataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.FileSystemFactoryOptions.html">pyarrow.dataset.FileSystemFactoryOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.FileSystemDatasetFactory.html">pyarrow.dataset.FileSystemDatasetFactory</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.UnionDataset.html">pyarrow.dataset.UnionDataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.Fragment.html">pyarrow.dataset.Fragment</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.FragmentScanOptions.html">pyarrow.dataset.FragmentScanOptions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.TaggedRecordBatch.html">pyarrow.dataset.TaggedRecordBatch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.Scanner.html">pyarrow.dataset.Scanner</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.Expression.html">pyarrow.dataset.Expression</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.InMemoryDataset.html">pyarrow.dataset.InMemoryDataset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.WrittenFile.html">pyarrow.dataset.WrittenFile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.dataset.get_partition_keys.html">pyarrow.dataset.get_partition_keys</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/cuda.html">CUDA Integration</a><input class="toctree-checkbox" id="toctree-checkbox-16" name="toctree-checkbox-16" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-16"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.Context.html">pyarrow.cuda.Context</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.CudaBuffer.html">pyarrow.cuda.CudaBuffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.new_host_buffer.html">pyarrow.cuda.new_host_buffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.HostBuffer.html">pyarrow.cuda.HostBuffer</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.BufferReader.html">pyarrow.cuda.BufferReader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.BufferWriter.html">pyarrow.cuda.BufferWriter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.serialize_record_batch.html">pyarrow.cuda.serialize_record_batch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.read_record_batch.html">pyarrow.cuda.read_record_batch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.read_message.html">pyarrow.cuda.read_message</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cuda.IpcMemHandle.html">pyarrow.cuda.IpcMemHandle</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api/misc.html">Miscellaneous</a><input class="toctree-checkbox" id="toctree-checkbox-17" name="toctree-checkbox-17" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-17"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.cpu_count.html">pyarrow.cpu_count</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.set_cpu_count.html">pyarrow.set_cpu_count</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.io_thread_count.html">pyarrow.io_thread_count</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.set_io_thread_count.html">pyarrow.set_io_thread_count</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.get_include.html">pyarrow.get_include</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.get_libraries.html">pyarrow.get_libraries</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="generated/pyarrow.get_library_dirs.html">pyarrow.get_library_dirs</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="getting_involved.html">Getting Involved</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="benchmarks.html">Benchmarks</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/cookbook/py/">Python cookbook</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| </div> |
| |
| <div id="rtd-footer-container"></div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| |
| |
| <nav aria-label="Breadcrumb"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Python</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page">Data Types...</li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="data-types-and-in-memory-data-model"> |
| <span id="data"></span><h1>Data Types and In-Memory Data Model<a class="headerlink" href="#data-types-and-in-memory-data-model" title="Permalink to this heading">#</a></h1> |
| <p>Apache Arrow defines columnar array data structures by composing type metadata |
| with memory buffers, like the ones explained in the documentation on |
| <a class="reference internal" href="memory.html#io"><span class="std std-ref">Memory and IO</span></a>. These data structures are exposed in Python through |
| a series of interrelated classes:</p> |
| <ul class="simple"> |
| <li><p><strong>Type Metadata</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.DataType</span></code>, which describe a logical |
| array type</p></li> |
| <li><p><strong>Schemas</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.Schema</span></code>, which describe a named |
| collection of types. These can be thought of as the column types in a |
| table-like object.</p></li> |
| <li><p><strong>Arrays</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.Array</span></code>, which are atomic, contiguous |
| columnar data structures composed from Arrow Buffer objects</p></li> |
| <li><p><strong>Record Batches</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>, which are a |
| collection of Array objects with a particular Schema</p></li> |
| <li><p><strong>Tables</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.Table</span></code>, a logical table data structure in |
| which each column consists of one or more <code class="docutils literal notranslate"><span class="pre">pyarrow.Array</span></code> objects of the |
| same type.</p></li> |
| </ul> |
| <p>We will examine these in the sections below in a series of examples.</p> |
| <section id="type-metadata"> |
| <span id="data-types"></span><h2>Type Metadata<a class="headerlink" href="#type-metadata" title="Permalink to this heading">#</a></h2> |
| <p>Apache Arrow defines language agnostic column-oriented data structures for |
| array data. These include:</p> |
| <ul class="simple"> |
| <li><p><strong>Fixed-length primitive types</strong>: numbers, booleans, date and times, fixed |
| size binary, decimals, and other values that fit into a given number</p></li> |
| <li><p><strong>Variable-length primitive types</strong>: binary, string</p></li> |
| <li><p><strong>Nested types</strong>: list, map, struct, and union</p></li> |
| <li><p><strong>Dictionary type</strong>: An encoded categorical type (more on this later)</p></li> |
| </ul> |
| <p>Each logical data type in Arrow has a corresponding factory function for |
| creating an instance of that type object in Python:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [1]: </span><span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="nn">pa</span> |
| |
| <span class="gp">In [2]: </span><span class="n">t1</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">()</span> |
| |
| <span class="gp">In [3]: </span><span class="n">t2</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">()</span> |
| |
| <span class="gp">In [4]: </span><span class="n">t3</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">binary</span><span class="p">()</span> |
| |
| <span class="gp">In [5]: </span><span class="n">t4</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">binary</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> |
| |
| <span class="gp">In [6]: </span><span class="n">t5</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'ms'</span><span class="p">)</span> |
| |
| <span class="gp">In [7]: </span><span class="n">t1</span> |
| <span class="gh">Out[7]: </span><span class="go">DataType(int32)</span> |
| |
| <span class="gp">In [8]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span> |
| <span class="go">int32</span> |
| |
| <span class="gp">In [9]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t4</span><span class="p">)</span> |
| <span class="go">fixed_size_binary[10]</span> |
| |
| <span class="gp">In [10]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t5</span><span class="p">)</span> |
| <span class="go">timestamp[ms]</span> |
| </pre></div> |
| </div> |
| <p>We use the name <strong>logical type</strong> because the <strong>physical</strong> storage may be the |
| same for one or more types. For example, <code class="docutils literal notranslate"><span class="pre">int64</span></code>, <code class="docutils literal notranslate"><span class="pre">float64</span></code>, and |
| <code class="docutils literal notranslate"><span class="pre">timestamp[ms]</span></code> all occupy 64 bits per value.</p> |
| <p>These objects are <cite>metadata</cite>; they are used for describing the data in arrays, |
| schemas, and record batches. In Python, they can be used in functions where the |
| input data (e.g. Python objects) may be coerced to more than one Arrow type.</p> |
| <p>The <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field" title="pyarrow.Field"><code class="xref py py-class docutils literal notranslate"><span class="pre">Field</span></code></a> type is a type plus a name and optional |
| user-defined metadata:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [11]: </span><span class="n">f0</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'int32_field'</span><span class="p">,</span> <span class="n">t1</span><span class="p">)</span> |
| |
| <span class="gp">In [12]: </span><span class="n">f0</span> |
| <span class="gh">Out[12]: </span><span class="go">pyarrow.Field<int32_field: int32></span> |
| |
| <span class="gp">In [13]: </span><span class="n">f0</span><span class="o">.</span><span class="n">name</span> |
| <span class="gh">Out[13]: </span><span class="go">'int32_field'</span> |
| |
| <span class="gp">In [14]: </span><span class="n">f0</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[14]: </span><span class="go">DataType(int32)</span> |
| </pre></div> |
| </div> |
| <p>Arrow supports <strong>nested value types</strong> like list, map, struct, and union. When |
| creating these, you must pass types or fields to indicate the data types of the |
| types’ children. For example, we can define a list of int32 values with:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [15]: </span><span class="n">t6</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">list_</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span> |
| |
| <span class="gp">In [16]: </span><span class="n">t6</span> |
| <span class="gh">Out[16]: </span><span class="go">ListType(list<item: int32>)</span> |
| </pre></div> |
| </div> |
| <p>A <cite>struct</cite> is a collection of named fields:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [17]: </span><span class="n">fields</span> <span class="o">=</span> <span class="p">[</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s0'</span><span class="p">,</span> <span class="n">t1</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s1'</span><span class="p">,</span> <span class="n">t2</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s2'</span><span class="p">,</span> <span class="n">t4</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">,</span> <span class="n">t6</span><span class="p">),</span> |
| <span class="gp"> ....: </span><span class="p">]</span> |
| <span class="gp"> ....: </span> |
| |
| <span class="gp">In [18]: </span><span class="n">t7</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">(</span><span class="n">fields</span><span class="p">)</span> |
| |
| <span class="gp">In [19]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t7</span><span class="p">)</span> |
| <span class="go">struct<s0: int32, s1: string, s2: fixed_size_binary[10], s3: list<item: int32>></span> |
| </pre></div> |
| </div> |
| <p>For convenience, you can pass <code class="docutils literal notranslate"><span class="pre">(name,</span> <span class="pre">type)</span></code> tuples directly instead of |
| <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field" title="pyarrow.Field"><code class="xref py py-class docutils literal notranslate"><span class="pre">Field</span></code></a> instances:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [20]: </span><span class="n">t8</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">([(</span><span class="s1">'s0'</span><span class="p">,</span> <span class="n">t1</span><span class="p">),</span> <span class="p">(</span><span class="s1">'s1'</span><span class="p">,</span> <span class="n">t2</span><span class="p">),</span> <span class="p">(</span><span class="s1">'s2'</span><span class="p">,</span> <span class="n">t4</span><span class="p">),</span> <span class="p">(</span><span class="s1">'s3'</span><span class="p">,</span> <span class="n">t6</span><span class="p">)])</span> |
| |
| <span class="gp">In [21]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t8</span><span class="p">)</span> |
| <span class="go">struct<s0: int32, s1: string, s2: fixed_size_binary[10], s3: list<item: int32>></span> |
| |
| <span class="gp">In [22]: </span><span class="n">t8</span> <span class="o">==</span> <span class="n">t7</span> |
| <span class="gh">Out[22]: </span><span class="go">True</span> |
| </pre></div> |
| </div> |
| <p>See <a class="reference internal" href="api/datatypes.html#api-types"><span class="std std-ref">Data Types API</span></a> for a full listing of data type |
| functions.</p> |
| </section> |
| <section id="schemas"> |
| <span id="data-schema"></span><h2>Schemas<a class="headerlink" href="#schemas" title="Permalink to this heading">#</a></h2> |
| <p>The <a class="reference internal" href="generated/pyarrow.Schema.html#pyarrow.Schema" title="pyarrow.Schema"><code class="xref py py-class docutils literal notranslate"><span class="pre">Schema</span></code></a> type is similar to the <code class="docutils literal notranslate"><span class="pre">struct</span></code> array type; it |
| defines the column names and types in a record batch or table data |
| structure. The <a class="reference internal" href="generated/pyarrow.schema.html#pyarrow.schema" title="pyarrow.schema"><code class="xref py py-func docutils literal notranslate"><span class="pre">pyarrow.schema()</span></code></a> factory function makes new Schema objects in |
| Python:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [23]: </span><span class="n">my_schema</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([(</span><span class="s1">'field0'</span><span class="p">,</span> <span class="n">t1</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'field1'</span><span class="p">,</span> <span class="n">t2</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'field2'</span><span class="p">,</span> <span class="n">t4</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'field3'</span><span class="p">,</span> <span class="n">t6</span><span class="p">)])</span> |
| <span class="gp"> ....: </span> |
| |
| <span class="gp">In [24]: </span><span class="n">my_schema</span> |
| <span class="gh">Out[24]: </span> |
| <span class="go">field0: int32</span> |
| <span class="go">field1: string</span> |
| <span class="go">field2: fixed_size_binary[10]</span> |
| <span class="go">field3: list<item: int32></span> |
| <span class="go"> child 0, item: int32</span> |
| </pre></div> |
| </div> |
| <p>In some applications, you may not create schemas directly, only using the ones |
| that are embedded in <a class="reference internal" href="ipc.html#ipc"><span class="std std-ref">IPC messages</span></a>.</p> |
| </section> |
| <section id="arrays"> |
| <span id="data-array"></span><h2>Arrays<a class="headerlink" href="#arrays" title="Permalink to this heading">#</a></h2> |
| <p>For each data type, there is an accompanying array data structure for holding |
| memory buffers that define a single contiguous chunk of columnar array |
| data. When you are using PyArrow, this data may come from IPC tools, though it |
| can also be created from various types of Python sequences (lists, NumPy |
| arrays, pandas data).</p> |
| <p>A simple way to create arrays is with <code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code>, which is similar to |
| the <code class="docutils literal notranslate"><span class="pre">numpy.array</span></code> function. By default PyArrow will infer the data type |
| for you:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [25]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span> |
| |
| <span class="gp">In [26]: </span><span class="n">arr</span> |
| <span class="gh">Out[26]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7fa1f1c7d8a0></span> |
| <span class="go">[</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> null,</span> |
| <span class="go"> 3</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>But you may also pass a specific data type to override type inference:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [27]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">uint16</span><span class="p">())</span> |
| <span class="gh">Out[27]: </span> |
| <span class="go"><pyarrow.lib.UInt16Array object at 0x7fa1f1c7dcc0></span> |
| <span class="go">[</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>The array’s <code class="docutils literal notranslate"><span class="pre">type</span></code> attribute is the corresponding piece of type metadata:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [28]: </span><span class="n">arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[28]: </span><span class="go">DataType(int64)</span> |
| </pre></div> |
| </div> |
| <p>Each in-memory array has a known length and null count (which will be 0 if |
| there are no null values):</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [29]: </span><span class="nb">len</span><span class="p">(</span><span class="n">arr</span><span class="p">)</span> |
| <span class="gh">Out[29]: </span><span class="go">4</span> |
| |
| <span class="gp">In [30]: </span><span class="n">arr</span><span class="o">.</span><span class="n">null_count</span> |
| <span class="gh">Out[30]: </span><span class="go">1</span> |
| </pre></div> |
| </div> |
| <p>Scalar values can be selected with normal indexing. <code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> converts |
| <code class="docutils literal notranslate"><span class="pre">None</span></code> values to Arrow nulls; we return the special <code class="docutils literal notranslate"><span class="pre">pyarrow.NA</span></code> value for |
| nulls:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [31]: </span><span class="n">arr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> |
| <span class="gh">Out[31]: </span><span class="go"><pyarrow.Int64Scalar: 1></span> |
| |
| <span class="gp">In [32]: </span><span class="n">arr</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> |
| <span class="gh">Out[32]: </span><span class="go"><pyarrow.Int64Scalar: None></span> |
| </pre></div> |
| </div> |
| <p>Arrow data is immutable, so values can be selected but not assigned.</p> |
| <p>Arrays can be sliced without copying:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [33]: </span><span class="n">arr</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="mi">3</span><span class="p">]</span> |
| <span class="gh">Out[33]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7fa1f1c7dfc0></span> |
| <span class="go">[</span> |
| <span class="go"> 2,</span> |
| <span class="go"> null</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <section id="none-values-and-nan-handling"> |
| <h3>None values and NAN handling<a class="headerlink" href="#none-values-and-nan-handling" title="Permalink to this heading">#</a></h3> |
| <p>As mentioned in the above section, the Python object <code class="docutils literal notranslate"><span class="pre">None</span></code> is always |
| converted to an Arrow null element on the conversion to <code class="docutils literal notranslate"><span class="pre">pyarrow.Array</span></code>. For |
| the float NaN value which is either represented by the Python object |
| <code class="docutils literal notranslate"><span class="pre">float('nan')</span></code> or <code class="docutils literal notranslate"><span class="pre">numpy.nan</span></code> we normally convert it to a <em>valid</em> float |
| value during the conversion. If an integer input is supplied to |
| <code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> that contains <code class="docutils literal notranslate"><span class="pre">np.nan</span></code>, <code class="docutils literal notranslate"><span class="pre">ValueError</span></code> is raised.</p> |
| <p>To handle better compatibility with Pandas, we support interpreting NaN values as |
| null elements. This is enabled automatically on all <code class="docutils literal notranslate"><span class="pre">from_pandas</span></code> function and |
| can be enabled on the other conversion functions by passing <code class="docutils literal notranslate"><span class="pre">from_pandas=True</span></code> |
| as a function parameter.</p> |
| </section> |
| <section id="list-arrays"> |
| <h3>List arrays<a class="headerlink" href="#list-arrays" title="Permalink to this heading">#</a></h3> |
| <p><code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> is able to infer the type of simple nested data structures |
| like lists:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [34]: </span><span class="n">nested_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([[],</span> <span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">]])</span> |
| |
| <span class="gp">In [35]: </span><span class="nb">print</span><span class="p">(</span><span class="n">nested_arr</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> |
| <span class="go">list<item: int64></span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="listview-arrays"> |
| <h3>ListView arrays<a class="headerlink" href="#listview-arrays" title="Permalink to this heading">#</a></h3> |
| <p><code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> can create an alternate list type called ListView:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [36]: </span><span class="n">nested_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([[],</span> <span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">]],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">list_view</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">()))</span> |
| |
| <span class="gp">In [37]: </span><span class="nb">print</span><span class="p">(</span><span class="n">nested_arr</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> |
| <span class="go">list_view<item: int64></span> |
| </pre></div> |
| </div> |
| <p>ListView arrays have a different set of buffers than List arrays. The ListView array |
| has both an offsets and sizes buffer, while a List array only has an offsets buffer. |
| This allows for ListView arrays to specify out-of-order offsets:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [38]: </span><span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]</span> |
| |
| <span class="gp">In [39]: </span><span class="n">offsets</span> <span class="o">=</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span> |
| |
| <span class="gp">In [40]: </span><span class="n">sizes</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span> |
| |
| <span class="gp">In [41]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">ListViewArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">offsets</span><span class="p">,</span> <span class="n">sizes</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span> |
| |
| <span class="gp">In [42]: </span><span class="n">arr</span> |
| <span class="gh">Out[42]: </span> |
| <span class="go"><pyarrow.lib.ListViewArray object at 0x7fa1f1c7e680></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>See the format specification for more details on <a class="reference internal" href="../format/Columnar.html#listview-layout"><span class="std std-ref">ListView Layout</span></a>.</p> |
| </section> |
| <section id="struct-arrays"> |
| <h3>Struct arrays<a class="headerlink" href="#struct-arrays" title="Permalink to this heading">#</a></h3> |
| <p><code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> is able to infer the schema of a struct type from arrays of |
| dictionaries:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [43]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span> <span class="p">{</span><span class="s1">'z'</span><span class="p">:</span> <span class="mf">3.4</span><span class="p">,</span> <span class="s1">'x'</span><span class="p">:</span> <span class="mi">4</span><span class="p">}])</span> |
| <span class="gh">Out[43]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7fa1f1c7e5c0></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int64</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> null</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 2 type: double</span> |
| <span class="go"> [</span> |
| <span class="go"> null,</span> |
| <span class="go"> 3.4</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>Struct arrays can be initialized from a sequence of Python dicts or tuples. For tuples, |
| you must explicitly pass the type:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [44]: </span><span class="n">ty</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">([(</span><span class="s1">'x'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">()),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'y'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">())])</span> |
| <span class="gp"> ....: </span> |
| |
| <span class="gp">In [45]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span> <span class="p">{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">:</span> <span class="kc">False</span><span class="p">}],</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[45]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7fa1f1c7eaa0></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int8</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> false</span> |
| <span class="go"> ]</span> |
| |
| <span class="gp">In [46]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([(</span><span class="mi">3</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span> <span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="kc">False</span><span class="p">)],</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[46]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7fa1f1c7eb60></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int8</span> |
| <span class="go"> [</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> false</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>When initializing a struct array, nulls are allowed both at the struct |
| level and at the individual field level. If initializing from a sequence |
| of Python dicts, a missing dict key is handled as a null value:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [47]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="kc">None</span><span class="p">,</span> <span class="p">{</span><span class="s1">'y'</span><span class="p">:</span> <span class="kc">None</span><span class="p">}],</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[47]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7fa1f1c7e740></span> |
| <span class="go">-- is_valid:</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> false,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 0 type: int8</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> null</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> null,</span> |
| <span class="go"> false,</span> |
| <span class="go"> null</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>You can also construct a struct array from existing arrays for each of the |
| struct’s components. In this case, data storage will be shared with the |
| individual arrays, and no copy is involved:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [48]: </span><span class="n">xs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int16</span><span class="p">())</span> |
| |
| <span class="gp">In [49]: </span><span class="n">ys</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| |
| <span class="gp">In [50]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">StructArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">((</span><span class="n">xs</span><span class="p">,</span> <span class="n">ys</span><span class="p">),</span> <span class="n">names</span><span class="o">=</span><span class="p">(</span><span class="s1">'x'</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">))</span> |
| |
| <span class="gp">In [51]: </span><span class="n">arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[51]: </span><span class="go">StructType(struct<x: int16, y: bool>)</span> |
| |
| <span class="gp">In [52]: </span><span class="n">arr</span> |
| <span class="gh">Out[52]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7fa1f1c7f280></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int16</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6,</span> |
| <span class="go"> 7</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> false,</span> |
| <span class="go"> true,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="map-arrays"> |
| <h3>Map arrays<a class="headerlink" href="#map-arrays" title="Permalink to this heading">#</a></h3> |
| <p>Map arrays can be constructed from lists of lists of tuples (key-item pairs), but only if |
| the type is explicitly passed into <a class="reference internal" href="generated/pyarrow.array.html#pyarrow.array" title="pyarrow.array"><code class="xref py py-meth docutils literal notranslate"><span class="pre">array()</span></code></a>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [53]: </span><span class="n">data</span> <span class="o">=</span> <span class="p">[[(</span><span class="s1">'x'</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="p">(</span><span class="s1">'y'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)],</span> <span class="p">[(</span><span class="s1">'a'</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="p">(</span><span class="s1">'b'</span><span class="p">,</span> <span class="mi">45</span><span class="p">)]]</span> |
| |
| <span class="gp">In [54]: </span><span class="n">ty</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">map_</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">())</span> |
| |
| <span class="gp">In [55]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[55]: </span> |
| <span class="go"><pyarrow.lib.MapArray object at 0x7fa1f1c7c9a0></span> |
| <span class="go">[</span> |
| <span class="go"> keys:</span> |
| <span class="go"> [</span> |
| <span class="go"> "x",</span> |
| <span class="go"> "y"</span> |
| <span class="go"> ]</span> |
| <span class="go"> values:</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0</span> |
| <span class="go"> ],</span> |
| <span class="go"> keys:</span> |
| <span class="go"> [</span> |
| <span class="go"> "a",</span> |
| <span class="go"> "b"</span> |
| <span class="go"> ]</span> |
| <span class="go"> values:</span> |
| <span class="go"> [</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 45</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>MapArrays can also be constructed from offset, key, and item arrays. Offsets represent the |
| starting position of each map. Note that the <a class="reference internal" href="generated/pyarrow.MapArray.html#pyarrow.MapArray.keys" title="pyarrow.MapArray.keys"><code class="xref py py-attr docutils literal notranslate"><span class="pre">MapArray.keys</span></code></a> and <a class="reference internal" href="generated/pyarrow.MapArray.html#pyarrow.MapArray.items" title="pyarrow.MapArray.items"><code class="xref py py-attr docutils literal notranslate"><span class="pre">MapArray.items</span></code></a> |
| properties give the <em>flattened</em> keys and items. To keep the keys and items associated to |
| their row, use the <a class="reference internal" href="generated/pyarrow.ListArray.html#pyarrow.ListArray.from_arrays" title="pyarrow.ListArray.from_arrays"><code class="xref py py-meth docutils literal notranslate"><span class="pre">ListArray.from_arrays()</span></code></a> constructor with the |
| <a class="reference internal" href="generated/pyarrow.MapArray.html#pyarrow.MapArray.offsets" title="pyarrow.MapArray.offsets"><code class="xref py py-attr docutils literal notranslate"><span class="pre">MapArray.offsets</span></code></a> property.</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [56]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">MapArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="s1">'x'</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">,</span> <span class="s1">'z'</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">])</span> |
| |
| <span class="gp">In [57]: </span><span class="n">arr</span><span class="o">.</span><span class="n">keys</span> |
| <span class="gh">Out[57]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7fa1f1c7f760></span> |
| <span class="go">[</span> |
| <span class="go"> "x",</span> |
| <span class="go"> "y",</span> |
| <span class="go"> "z"</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [58]: </span><span class="n">arr</span><span class="o">.</span><span class="n">items</span> |
| <span class="gh">Out[58]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7fa1f1c7f6a0></span> |
| <span class="go">[</span> |
| <span class="go"> 4,</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [59]: </span><span class="n">pa</span><span class="o">.</span><span class="n">ListArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">offsets</span><span class="p">,</span> <span class="n">arr</span><span class="o">.</span><span class="n">keys</span><span class="p">)</span> |
| <span class="gh">Out[59]: </span> |
| <span class="go"><pyarrow.lib.ListArray object at 0x7fa1f1c7f940></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> "x",</span> |
| <span class="go"> "y"</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> "z"</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [60]: </span><span class="n">pa</span><span class="o">.</span><span class="n">ListArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">offsets</span><span class="p">,</span> <span class="n">arr</span><span class="o">.</span><span class="n">items</span><span class="p">)</span> |
| <span class="gh">Out[60]: </span> |
| <span class="go"><pyarrow.lib.ListArray object at 0x7fa1f1c7fe80></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> 4,</span> |
| <span class="go"> 5</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 6</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="union-arrays"> |
| <h3>Union arrays<a class="headerlink" href="#union-arrays" title="Permalink to this heading">#</a></h3> |
| <p>The union type represents a nested array type where each value can be one |
| (and only one) of a set of possible types. There are two possible |
| storage types for union arrays: sparse and dense.</p> |
| <p>In a sparse union array, each of the child arrays has the same length |
| as the resulting union array. They are adjuncted with a <code class="docutils literal notranslate"><span class="pre">int8</span></code> “types” |
| array that tells, for each value, from which child array it must be |
| selected:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [61]: </span><span class="n">xs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">])</span> |
| |
| <span class="gp">In [62]: </span><span class="n">ys</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| |
| <span class="gp">In [63]: </span><span class="n">types</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">())</span> |
| |
| <span class="gp">In [64]: </span><span class="n">union_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">UnionArray</span><span class="o">.</span><span class="n">from_sparse</span><span class="p">(</span><span class="n">types</span><span class="p">,</span> <span class="p">[</span><span class="n">xs</span><span class="p">,</span> <span class="n">ys</span><span class="p">])</span> |
| |
| <span class="gp">In [65]: </span><span class="n">union_arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[65]: </span><span class="go">SparseUnionType(sparse_union<0: int64=0, 1: bool=1>)</span> |
| |
| <span class="gp">In [66]: </span><span class="n">union_arr</span> |
| <span class="gh">Out[66]: </span> |
| <span class="go"><pyarrow.lib.UnionArray object at 0x7fa1f1c7f040></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- type_ids: [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 1</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 0 type: int64</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6,</span> |
| <span class="go"> 7</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> false,</span> |
| <span class="go"> false,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>In a dense union array, you also pass, in addition to the <code class="docutils literal notranslate"><span class="pre">int8</span></code> “types” |
| array, a <code class="docutils literal notranslate"><span class="pre">int32</span></code> “offsets” array that tells, for each value, at |
| each offset in the selected child array it can be found:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [67]: </span><span class="n">xs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">])</span> |
| |
| <span class="gp">In [68]: </span><span class="n">ys</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| |
| <span class="gp">In [69]: </span><span class="n">types</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">())</span> |
| |
| <span class="gp">In [70]: </span><span class="n">offsets</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">())</span> |
| |
| <span class="gp">In [71]: </span><span class="n">union_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">UnionArray</span><span class="o">.</span><span class="n">from_dense</span><span class="p">(</span><span class="n">types</span><span class="p">,</span> <span class="n">offsets</span><span class="p">,</span> <span class="p">[</span><span class="n">xs</span><span class="p">,</span> <span class="n">ys</span><span class="p">])</span> |
| |
| <span class="gp">In [72]: </span><span class="n">union_arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[72]: </span><span class="go">DenseUnionType(dense_union<0: int64=0, 1: bool=1>)</span> |
| |
| <span class="gp">In [73]: </span><span class="n">union_arr</span> |
| <span class="gh">Out[73]: </span> |
| <span class="go"><pyarrow.lib.UnionArray object at 0x7fa1f1cf42e0></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- type_ids: [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 0</span> |
| <span class="go"> ]</span> |
| <span class="go">-- value_offsets: [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 0 type: int64</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6,</span> |
| <span class="go"> 7</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> false,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="dictionary-arrays"> |
| <span id="data-dictionary"></span><h3>Dictionary Arrays<a class="headerlink" href="#dictionary-arrays" title="Permalink to this heading">#</a></h3> |
| <p>The <strong>Dictionary</strong> type in PyArrow is a special array type that is similar to a |
| factor in R or a <code class="docutils literal notranslate"><span class="pre">pandas.Categorical</span></code>. It enables one or more record batches |
| in a file or stream to transmit integer <em>indices</em> referencing a shared |
| <strong>dictionary</strong> containing the distinct values in the logical array. This is |
| particularly often used with strings to save memory and improve performance.</p> |
| <p>The way that dictionaries are handled in the Apache Arrow format and the way |
| they appear in C++ and Python is slightly different. We define a special |
| <a class="reference internal" href="generated/pyarrow.DictionaryArray.html#pyarrow.DictionaryArray" title="pyarrow.DictionaryArray"><code class="xref py py-class docutils literal notranslate"><span class="pre">DictionaryArray</span></code></a> type with a corresponding dictionary type. Let’s |
| consider an example:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [74]: </span><span class="n">indices</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span> |
| |
| <span class="gp">In [75]: </span><span class="n">dictionary</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">])</span> |
| |
| <span class="gp">In [76]: </span><span class="n">dict_array</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">DictionaryArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dictionary</span><span class="p">)</span> |
| |
| <span class="gp">In [77]: </span><span class="n">dict_array</span> |
| <span class="gh">Out[77]: </span> |
| <span class="go"><pyarrow.lib.DictionaryArray object at 0x7fa1f1cdb140></span> |
| |
| <span class="go">-- dictionary:</span> |
| <span class="go"> [</span> |
| <span class="go"> "foo",</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz"</span> |
| <span class="go"> ]</span> |
| <span class="go">-- indices:</span> |
| <span class="go"> [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> null,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>Here we have:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [78]: </span><span class="nb">print</span><span class="p">(</span><span class="n">dict_array</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> |
| <span class="go">dictionary<values=string, indices=int64, ordered=0></span> |
| |
| <span class="gp">In [79]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">indices</span> |
| <span class="gh">Out[79]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7fa1f1cf4640></span> |
| <span class="go">[</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> null,</span> |
| <span class="go"> 2</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [80]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">dictionary</span> |
| <span class="gh">Out[80]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7fa1f1cf4100></span> |
| <span class="go">[</span> |
| <span class="go"> "foo",</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz"</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>When using <a class="reference internal" href="generated/pyarrow.DictionaryArray.html#pyarrow.DictionaryArray" title="pyarrow.DictionaryArray"><code class="xref py py-class docutils literal notranslate"><span class="pre">DictionaryArray</span></code></a> with pandas, the analogue is |
| <code class="docutils literal notranslate"><span class="pre">pandas.Categorical</span></code> (more on this later):</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [81]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span> |
| <span class="gh">Out[81]: </span> |
| <span class="go">0 foo</span> |
| <span class="go">1 bar</span> |
| <span class="go">2 foo</span> |
| <span class="go">3 bar</span> |
| <span class="go">4 baz</span> |
| <span class="go">5 foo</span> |
| <span class="go">6 NaN</span> |
| <span class="go">7 baz</span> |
| <span class="go">dtype: category</span> |
| <span class="go">Categories (3, object): ['foo', 'bar', 'baz']</span> |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| <section id="record-batches"> |
| <span id="data-record-batch"></span><h2>Record Batches<a class="headerlink" href="#record-batches" title="Permalink to this heading">#</a></h2> |
| <p>A <strong>Record Batch</strong> in Apache Arrow is a collection of equal-length array |
| instances. Let’s consider a collection of arrays:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [82]: </span><span class="n">data</span> <span class="o">=</span> <span class="p">[</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">,</span> <span class="kc">None</span><span class="p">]),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">True</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| <span class="gp"> ....: </span><span class="p">]</span> |
| <span class="gp"> ....: </span> |
| </pre></div> |
| </div> |
| <p>A record batch can be created from this list of arrays using |
| <code class="docutils literal notranslate"><span class="pre">RecordBatch.from_arrays</span></code>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [83]: </span><span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">[</span><span class="s1">'f0'</span><span class="p">,</span> <span class="s1">'f1'</span><span class="p">,</span> <span class="s1">'f2'</span><span class="p">])</span> |
| |
| <span class="gp">In [84]: </span><span class="n">batch</span><span class="o">.</span><span class="n">num_columns</span> |
| <span class="gh">Out[84]: </span><span class="go">3</span> |
| |
| <span class="gp">In [85]: </span><span class="n">batch</span><span class="o">.</span><span class="n">num_rows</span> |
| <span class="gh">Out[85]: </span><span class="go">4</span> |
| |
| <span class="gp">In [86]: </span><span class="n">batch</span><span class="o">.</span><span class="n">schema</span> |
| <span class="gh">Out[86]: </span> |
| <span class="go">f0: int64</span> |
| <span class="go">f1: string</span> |
| <span class="go">f2: bool</span> |
| |
| <span class="gp">In [87]: </span><span class="n">batch</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> |
| <span class="gh">Out[87]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7fa1f1cf4dc0></span> |
| <span class="go">[</span> |
| <span class="go"> "foo",</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz",</span> |
| <span class="go"> null</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>A record batch can be sliced without copying memory like an array:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [88]: </span><span class="n">batch2</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">slice</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span> |
| |
| <span class="gp">In [89]: </span><span class="n">batch2</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> |
| <span class="gh">Out[89]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7fa1f1cf4fa0></span> |
| <span class="go">[</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz",</span> |
| <span class="go"> null</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="tables"> |
| <span id="data-table"></span><h2>Tables<a class="headerlink" href="#tables" title="Permalink to this heading">#</a></h2> |
| <p>The PyArrow <a class="reference internal" href="generated/pyarrow.Table.html#pyarrow.Table" title="pyarrow.Table"><code class="xref py py-class docutils literal notranslate"><span class="pre">Table</span></code></a> type is not part of the Apache Arrow |
| specification, but is rather a tool to help with wrangling multiple record |
| batches and array pieces as a single logical dataset. As a relevant example, we |
| may receive multiple small record batches in a socket stream, then need to |
| concatenate them into contiguous memory for use in NumPy or pandas. The Table |
| object makes this efficient without requiring additional memory copying.</p> |
| <p>Considering the record batch we created above, we can create a Table containing |
| one or more copies of the batch using <code class="docutils literal notranslate"><span class="pre">Table.from_batches</span></code>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [90]: </span><span class="n">batches</span> <span class="o">=</span> <span class="p">[</span><span class="n">batch</span><span class="p">]</span> <span class="o">*</span> <span class="mi">5</span> |
| |
| <span class="gp">In [91]: </span><span class="n">table</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="o">.</span><span class="n">from_batches</span><span class="p">(</span><span class="n">batches</span><span class="p">)</span> |
| |
| <span class="gp">In [92]: </span><span class="n">table</span> |
| <span class="gh">Out[92]: </span> |
| <span class="go">pyarrow.Table</span> |
| <span class="go">f0: int64</span> |
| <span class="go">f1: string</span> |
| <span class="go">f2: bool</span> |
| <span class="gt">----</span> |
| <span class="ne">f0</span>: [[1,2,3,4],[1,2,3,4],...,[1,2,3,4],[1,2,3,4]] |
| <span class="ne">f1</span>: [["foo","bar","baz",null],["foo","bar","baz",null],...,["foo","bar","baz",null],["foo","bar","baz",null]] |
| <span class="ne">f2</span>: [[true,null,false,true],[true,null,false,true],...,[true,null,false,true],[true,null,false,true]] |
| |
| <span class="gp">In [93]: </span><span class="n">table</span><span class="o">.</span><span class="n">num_rows</span> |
| <span class="gh">Out[93]: </span><span class="go">20</span> |
| </pre></div> |
| </div> |
| <p>The table’s columns are instances of <a class="reference internal" href="generated/pyarrow.ChunkedArray.html#pyarrow.ChunkedArray" title="pyarrow.ChunkedArray"><code class="xref py py-class docutils literal notranslate"><span class="pre">ChunkedArray</span></code></a>, which is a |
| container for one or more arrays of the same type.</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [94]: </span><span class="n">c</span> <span class="o">=</span> <span class="n">table</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> |
| |
| <span class="gp">In [95]: </span><span class="n">c</span> |
| <span class="gh">Out[95]: </span> |
| <span class="go"><pyarrow.lib.ChunkedArray object at 0x7fa1f1d2d710></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go">...,</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [96]: </span><span class="n">c</span><span class="o">.</span><span class="n">num_chunks</span> |
| <span class="gh">Out[96]: </span><span class="go">5</span> |
| |
| <span class="gp">In [97]: </span><span class="n">c</span><span class="o">.</span><span class="n">chunk</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> |
| <span class="gh">Out[97]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7fa1f1cf5180></span> |
| <span class="go">[</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>As you’ll see in the <a class="reference internal" href="pandas.html#pandas-interop"><span class="std std-ref">pandas section</span></a>, we can convert |
| these objects to contiguous NumPy arrays for use in pandas:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [98]: </span><span class="n">c</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span> |
| <span class="gh">Out[98]: </span> |
| <span class="go">0 1</span> |
| <span class="go">1 2</span> |
| <span class="go">2 3</span> |
| <span class="go">3 4</span> |
| <span class="go">4 1</span> |
| <span class="go">5 2</span> |
| <span class="go">6 3</span> |
| <span class="go">7 4</span> |
| <span class="go">8 1</span> |
| <span class="go">9 2</span> |
| <span class="go">10 3</span> |
| <span class="go">11 4</span> |
| <span class="go">12 1</span> |
| <span class="go">13 2</span> |
| <span class="go">14 3</span> |
| <span class="go">15 4</span> |
| <span class="go">16 1</span> |
| <span class="go">17 2</span> |
| <span class="go">18 3</span> |
| <span class="go">19 4</span> |
| <span class="go">Name: f0, dtype: int64</span> |
| </pre></div> |
| </div> |
| <p>Multiple tables can also be concatenated together to form a single table using |
| <code class="docutils literal notranslate"><span class="pre">pyarrow.concat_tables</span></code>, if the schemas are equal:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [99]: </span><span class="n">tables</span> <span class="o">=</span> <span class="p">[</span><span class="n">table</span><span class="p">]</span> <span class="o">*</span> <span class="mi">2</span> |
| |
| <span class="gp">In [100]: </span><span class="n">table_all</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">concat_tables</span><span class="p">(</span><span class="n">tables</span><span class="p">)</span> |
| |
| <span class="gp">In [101]: </span><span class="n">table_all</span><span class="o">.</span><span class="n">num_rows</span> |
| <span class="gh">Out[101]: </span><span class="go">40</span> |
| |
| <span class="gp">In [102]: </span><span class="n">c</span> <span class="o">=</span> <span class="n">table_all</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> |
| |
| <span class="gp">In [103]: </span><span class="n">c</span><span class="o">.</span><span class="n">num_chunks</span> |
| <span class="gh">Out[103]: </span><span class="go">10</span> |
| </pre></div> |
| </div> |
| <p>This is similar to <code class="docutils literal notranslate"><span class="pre">Table.from_batches</span></code>, but uses tables as input instead of |
| record batches. Record batches can be made into tables, but not the other way |
| around, so if your data is already in table form, then use |
| <code class="docutils literal notranslate"><span class="pre">pyarrow.concat_tables</span></code>.</p> |
| </section> |
| <section id="custom-schema-and-field-metadata"> |
| <h2>Custom Schema and Field Metadata<a class="headerlink" href="#custom-schema-and-field-metadata" title="Permalink to this heading">#</a></h2> |
| <p>Arrow supports both schema-level and field-level custom key-value metadata |
| allowing for systems to insert their own application defined metadata to |
| customize behavior.</p> |
| <p>Custom metadata can be accessed at <a class="reference internal" href="generated/pyarrow.Schema.html#pyarrow.Schema.metadata" title="pyarrow.Schema.metadata"><code class="xref py py-attr docutils literal notranslate"><span class="pre">Schema.metadata</span></code></a> for the schema-level |
| and <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field.metadata" title="pyarrow.Field.metadata"><code class="xref py py-attr docutils literal notranslate"><span class="pre">Field.metadata</span></code></a> for the field-level.</p> |
| <p>Note that this metadata is preserved in <a class="reference internal" href="ipc.html#ipc"><span class="std std-ref">Streaming, Serialization, and IPC</span></a> processes.</p> |
| <p>To customize the schema metadata of an existing table you can use |
| <a class="reference internal" href="generated/pyarrow.Table.html#pyarrow.Table.replace_schema_metadata" title="pyarrow.Table.replace_schema_metadata"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Table.replace_schema_metadata()</span></code></a>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [104]: </span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">metadata</span> <span class="c1"># empty</span> |
| |
| <span class="gp">In [105]: </span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">replace_schema_metadata</span><span class="p">({</span><span class="s2">"f0"</span><span class="p">:</span> <span class="s2">"First dose"</span><span class="p">})</span> |
| |
| <span class="gp">In [106]: </span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[106]: </span><span class="go">{b'f0': b'First dose'}</span> |
| </pre></div> |
| </div> |
| <p>To customize the metadata of the field from the table schema you can use |
| <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field.with_metadata" title="pyarrow.Field.with_metadata"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Field.with_metadata()</span></code></a>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [107]: </span><span class="n">field_f1</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"f1"</span><span class="p">)</span> |
| |
| <span class="gp">In [108]: </span><span class="n">field_f1</span><span class="o">.</span><span class="n">metadata</span> <span class="c1"># empty</span> |
| |
| <span class="gp">In [109]: </span><span class="n">field_f1</span> <span class="o">=</span> <span class="n">field_f1</span><span class="o">.</span><span class="n">with_metadata</span><span class="p">({</span><span class="s2">"f1"</span><span class="p">:</span> <span class="s2">"Second dose"</span><span class="p">})</span> |
| |
| <span class="gp">In [110]: </span><span class="n">field_f1</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[110]: </span><span class="go">{b'f1': b'Second dose'}</span> |
| </pre></div> |
| </div> |
| <p>Both options create a shallow copy of the data and do not in fact change the |
| Schema which is immutable. To change the metadata in the schema of the table |
| we created a new object when calling <a class="reference internal" href="generated/pyarrow.Table.html#pyarrow.Table.replace_schema_metadata" title="pyarrow.Table.replace_schema_metadata"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Table.replace_schema_metadata()</span></code></a>.</p> |
| <p>To change the metadata of the field in the schema we would need to define |
| a new schema and cast the data to this schema:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [111]: </span><span class="n">my_schema2</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([</span> |
| <span class="gp"> .....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'f0'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"First dose"</span><span class="p">}),</span> |
| <span class="gp"> .....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'f1'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">(),</span> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"Second dose"</span><span class="p">}),</span> |
| <span class="gp"> .....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'f2'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">())],</span> |
| <span class="gp"> .....: </span> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"f2"</span><span class="p">:</span> <span class="s2">"booster"</span><span class="p">})</span> |
| <span class="gp"> .....: </span> |
| |
| <span class="gp">In [112]: </span><span class="n">t2</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">my_schema2</span><span class="p">)</span> |
| |
| <span class="gp">In [113]: </span><span class="n">t2</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"f0"</span><span class="p">)</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[113]: </span><span class="go">{b'name': b'First dose'}</span> |
| |
| <span class="gp">In [114]: </span><span class="n">t2</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"f1"</span><span class="p">)</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[114]: </span><span class="go">{b'name': b'Second dose'}</span> |
| |
| <span class="gp">In [115]: </span><span class="n">t2</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[115]: </span><span class="go">{b'f2': b'booster'}</span> |
| </pre></div> |
| </div> |
| <p>Metadata key and value pairs are <code class="docutils literal notranslate"><span class="pre">std::string</span></code> objects in the C++ implementation |
| and so they are bytes objects (<code class="docutils literal notranslate"><span class="pre">b'...'</span></code>) in Python.</p> |
| </section> |
| <section id="record-batch-readers"> |
| <h2>Record Batch Readers<a class="headerlink" href="#record-batch-readers" title="Permalink to this heading">#</a></h2> |
| <p>Many functions in PyArrow either return or take as an argument a <a class="reference internal" href="generated/pyarrow.RecordBatchReader.html#pyarrow.RecordBatchReader" title="pyarrow.RecordBatchReader"><code class="xref py py-class docutils literal notranslate"><span class="pre">RecordBatchReader</span></code></a>. |
| It can be used like any iterable of record batches, but also provides their common |
| schema without having to get any of the batches.:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">schema</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([(</span><span class="s1">'x'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">())])</span> |
| <span class="gp">>>> </span><span class="k">def</span> <span class="nf">iter_record_batches</span><span class="p">():</span> |
| <span class="gp">... </span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">2</span><span class="p">):</span> |
| <span class="gp">... </span> <span class="k">yield</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">([</span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])],</span> <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">reader</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchReader</span><span class="o">.</span><span class="n">from_batches</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">iter_record_batches</span><span class="p">())</span> |
| <span class="gp">>>> </span><span class="nb">print</span><span class="p">(</span><span class="n">reader</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> |
| <span class="go">pyarrow.Schema</span> |
| <span class="go">x: int64</span> |
| <span class="gp">>>> </span><span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">reader</span><span class="p">:</span> |
| <span class="gp">... </span> <span class="nb">print</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span> |
| <span class="go">pyarrow.RecordBatch</span> |
| <span class="go">x: int64</span> |
| <span class="go">pyarrow.RecordBatch</span> |
| <span class="go">x: int64</span> |
| </pre></div> |
| </div> |
| <p>It can also be sent between languages using the <a class="reference internal" href="../format/CStreamInterface.html#c-stream-interface"><span class="std std-ref">C stream interface</span></a>.</p> |
| </section> |
| <section id="conversion-of-recordbatch-do-tensor"> |
| <h2>Conversion of RecordBatch do Tensor<a class="headerlink" href="#conversion-of-recordbatch-do-tensor" title="Permalink to this heading">#</a></h2> |
| <p>Each array of the <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> has it’s own contiguous memory that is not necessarily |
| adjacent to other arrays. A different memory structure that is used in machine learning |
| libraries is a two dimensional array (also called a 2-dim tensor or a matrix) which takes |
| only one contiguous block of memory.</p> |
| <p>For this reason there is a function <code class="docutils literal notranslate"><span class="pre">pyarrow.RecordBatch.to_tensor()</span></code> available |
| to efficiently convert tabular columnar data into a tensor.</p> |
| <p>Data types supported in this conversion are unsigned, signed integer and float |
| types. Currently only column-major conversion is supported.</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span> <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="nn">pa</span> |
| <span class="gp">>>> </span> <span class="n">arr1</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">]</span> |
| <span class="gp">>>> </span> <span class="n">arr2</span> <span class="o">=</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">,</span> <span class="mi">50</span><span class="p">]</span> |
| <span class="gp">>>> </span> <span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">arr1</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">uint16</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">arr2</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int16</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="p">],</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">]</span> |
| <span class="gp">... </span> <span class="p">)</span> |
| <span class="gp">>>> </span> <span class="n">batch</span><span class="o">.</span><span class="n">to_tensor</span><span class="p">()</span> |
| <span class="go"><pyarrow.Tensor></span> |
| <span class="go">type: int32</span> |
| <span class="go">shape: (9, 2)</span> |
| <span class="go">strides: (4, 36)</span> |
| <span class="gp">>>> </span> <span class="n">batch</span><span class="o">.</span><span class="n">to_tensor</span><span class="p">()</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span> |
| <span class="go">array([[ 1, 10],</span> |
| <span class="go"> [ 2, 20],</span> |
| <span class="go"> [ 3, 30],</span> |
| <span class="go"> [ 4, 40],</span> |
| <span class="go"> [ 5, 50]], dtype=int32)</span> |
| </pre></div> |
| </div> |
| <p>With <code class="docutils literal notranslate"><span class="pre">null_to_nan</span></code> set to <code class="docutils literal notranslate"><span class="pre">True</span></code> one can also convert data with |
| nulls. They will be converted to <code class="docutils literal notranslate"><span class="pre">NaN</span></code>:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">float32</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="p">],</span> <span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">]</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">batch</span><span class="o">.</span><span class="n">to_tensor</span><span class="p">(</span><span class="n">null_to_nan</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span> |
| <span class="go">array([[ 1., 10.],</span> |
| <span class="go"> [ 2., 20.],</span> |
| <span class="go"> [ 3., 30.],</span> |
| <span class="go"> [ 4., 40.],</span> |
| <span class="go"> [nan, nan]])</span> |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="getstarted.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Getting Started</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="compute.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Compute Functions</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#type-metadata">Type Metadata</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#schemas">Schemas</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#arrays">Arrays</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#none-values-and-nan-handling">None values and NAN handling</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#list-arrays">List arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#listview-arrays">ListView arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#struct-arrays">Struct arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#map-arrays">Map arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#union-arrays">Union arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#dictionary-arrays">Dictionary Arrays</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#record-batches">Record Batches</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#tables">Tables</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#custom-schema-and-field-metadata">Custom Schema and Field Metadata</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#record-batch-readers">Record Batch Readers</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#conversion-of-recordbatch-do-tensor">Conversion of RecordBatch do Tensor</a></li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/python/data.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script> |
| <script src="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2024 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
