| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <title>Data Types and In-Memory Data Model — Apache Arrow v22.0.0.dev115</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || ""; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=03e43079" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" /> |
| <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=8dcd28dc" /> |
| |
| <!-- So that users can add custom icons --> |
| <script src="../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" /> |
| |
| <script src="../_static/documentation_options.js?v=ae2a2051"></script> |
| <script src="../_static/doctools.js?v=9bcbadda"></script> |
| <script src="../_static/sphinx_highlight.js?v=dc90522c"></script> |
| <script src="../_static/clipboard.min.js?v=a7894cd8"></script> |
| <script src="../_static/copybutton.js?v=3bb21c8c"></script> |
| <script src="../_static/design-tabs.js?v=f930bc37"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'python/data';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.16.1'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = |
| true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/python/data.html" /> |
| <link rel="icon" href="../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Compute Functions" href="compute.html" /> |
| <link rel="prev" title="Getting Started" href="getstarted.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="22.0.0.dev115" /> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class=" navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v22.0.0.dev115 - Home"/> |
| <img src="../_static/arrow-dark.png" class="logo__image only-dark pst-js-only" alt="Apache Arrow v22.0.0.dev115 - Home"/> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class=" navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> |
| <span class="fa-solid fa-outdent"></span> |
| </button> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../cpp/index.html">C++</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../cpp/getting_started.html">Getting Started</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/build_system.html">Using Arrow C++ in your own project</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/conventions.html">Conventions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/tutorials/basic_arrow.html">Basic Arrow Data Structures</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/tutorials/io_tutorial.html">Arrow File I/O</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/tutorials/compute_tutorial.html">Arrow Compute</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/tutorials/datasets_tutorial.html">Arrow Datasets</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../cpp/user_guide.html">User Guide</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/overview.html">High-Level Overview</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/memory.html">Memory Management</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/arrays.html">Arrays</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/datatypes.html">Data Types</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/tables.html">Tabular Data</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/compute.html">Compute Functions</a></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../cpp/gandiva.html">The Gandiva Expression Compiler</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="../cpp/gandiva/expr_projector_filter.html">Gandiva Expression, Projector, and Filter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="../cpp/gandiva/external_func.html">Gandiva External Functions Development Guide</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../cpp/acero.html">Acero: A C++ streaming execution engine</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="../cpp/acero/overview.html">Acero Overview</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="../cpp/acero/user_guide.html">Acero User’s Guide</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="../cpp/acero/substrait.html">Using Acero with Substrait</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/io.html">Input / output and filesystems</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/ipc.html">Reading and writing the Arrow IPC format</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/orc.html">Reading and Writing ORC files</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/parquet.html">Reading and writing Parquet files</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/csv.html">Reading and Writing CSV files</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/json.html">Reading JSON files</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/dataset.html">Tabular Datasets</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/gdb.html">Debugging code using Arrow</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/threading.html">Thread Management</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/opentelemetry.html">OpenTelemetry</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/env_vars.html">Environment Variables</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../cpp/examples/index.html">Examples</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/examples/cmake_minimal_build.html">Minimal build using CMake</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/examples/compute_and_write_example.html">Compute and Write CSV Example</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/examples/dataset_documentation_example.html">Arrow Datasets example</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/examples/parquet_column_encryption.html">Parquet column encryption</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/examples/row_columnar_conversion.html">Row to columnar conversion</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/examples/tuple_range_conversion.html">std::tuple-like ranges to Arrow</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/examples/converting_recordbatch_to_tensor.html">Converting RecordBatch to Tensor</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../cpp/api.html">API Reference</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/support.html">Programming Support</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/memory.html">Memory (management)</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/thread.html">Thread (management)</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/datatype.html">Data Types</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/array.html">Arrays</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/extension.html">Extension</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/scalar.html">Scalars</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/builder.html">Array Builders</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/table.html">Two-dimensional Datasets</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/c_abi.html">C Interfaces</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/compute.html">Compute Functions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/acero.html">Streaming Execution (Acero)</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/gandiva.html">Gandiva Expression Compiler</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/tensor.html">Tensors</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/utilities.html">Utilities</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/async.html">Asynchronous programming</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/io.html">Input / output</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/ipc.html">Arrow IPC</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/formats.html">File Formats</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/cuda.html">CUDA support</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/flightsql.html">Arrow Flight SQL</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/filesystem.html">Filesystems</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/api/dataset.html">Dataset</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../developers/cpp/index.html">C++ Development</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../developers/cpp/building.html">Building Arrow C++</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../developers/cpp/development.html">Development Guidelines</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../developers/cpp/windows.html">Developing on Windows</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../developers/cpp/emscripten.html">Cross compiling for WebAssembly with Emscripten</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../developers/cpp/conventions.html">Conventions</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../developers/cpp/fuzzing.html">Fuzzing Arrow C++</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../developers/cpp/compute.html">Developing Arrow C++ Compute</a></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="../developers/cpp/acero.html">Developing Acero</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="../developers/cpp/acero/swiss_table.html">Swiss Table</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference external" href="https://arrow.apache.org/cookbook/cpp/">C++ Cookbook</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../c_glib/index.html">C GLib</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../c_glib/arrow-glib/index.html">Apache Arrow GLib</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../c_glib/arrow-cuda-glib/index.html">Apache Arrow CUDA GLib</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../c_glib/arrow-dataset-glib/index.html">Apache Arrow Dataset</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../c_glib/arrow-flight-glib/index.html">Apache Arrow Flight GLib</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../c_glib/arrow-flight-sql-glib/index.html">Apache Arrow Flight SQL GLib</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../c_glib/parquet-glib/index.html">Apache Parquet GLib</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../c_glib/gandiva-glib/index.html">Gandiva GLib</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/main/csharp/README.md">C#</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/go/">Go</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../java/index.html">Java</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../java/quickstartguide.html">Quick Start Guide</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/overview.html">High-Level Overview</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/install.html">Installing Java Modules</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/memory.html">Memory Management</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/vector.html">ValueVector</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/vector_schema_root.html">Tabular Data</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/table.html">Table</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/ipc.html">Reading/Writing IPC formats</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/algorithm.html">Java Algorithms</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/flight_sql.html">Arrow Flight SQL</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/flight_sql_jdbc_driver.html">Arrow Flight SQL JDBC Driver</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/dataset.html">Dataset</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/substrait.html">Substrait</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/cdata.html">C Data Interface</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/jdbc.html">Arrow JDBC Adapter</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/reference/index.html">Reference (javadoc)</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://arrow.apache.org/cookbook/java/">Java cookbook</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/js/current/">JavaScript</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/julia/">Julia</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/main/matlab/README.md">MATLAB</a></li> |
| <li class="toctree-l1 current active has-children"><a class="reference internal" href="index.html">Python</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l2"><a class="reference internal" href="install.html">Installing PyArrow</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="getstarted.html">Getting Started</a></li> |
| <li class="toctree-l2 current active"><a class="current reference internal" href="#">Data Types and In-Memory Data Model</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="compute.html">Compute Functions</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="memory.html">Memory and IO Interfaces</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="ipc.html">Streaming, Serialization, and IPC</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="filesystems.html">Filesystem Interface</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="numpy.html">NumPy Integration</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="pandas.html">Pandas Integration</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="interchange_protocol.html">Dataframe Interchange Protocol</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="dlpack.html">The DLPack Protocol</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="timestamps.html">Timestamps</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="orc.html">Reading and Writing the Apache ORC Format</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="csv.html">Reading and Writing CSV files</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="feather.html">Feather File Format</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="json.html">Reading JSON files</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="parquet.html">Reading and Writing the Apache Parquet Format</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="dataset.html">Tabular Datasets</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="extending_types.html">Extending PyArrow</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="integration.html">PyArrow Integrations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="integration/substrait.html">Substrait</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="integration/python_r.html">Integrating PyArrow with R</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="integration/python_java.html">Integrating PyArrow with Java</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="integration/extending.html">Using pyarrow from C++ and Cython Code</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="integration/cuda.html">CUDA Integration</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="env_vars.html">Environment Variables</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="api.html">API Reference</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/datatypes.html">Data Types and Schemas</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.null.html">pyarrow.null</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.bool_.html">pyarrow.bool_</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.int8.html">pyarrow.int8</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.int16.html">pyarrow.int16</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.int32.html">pyarrow.int32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.int64.html">pyarrow.int64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.uint8.html">pyarrow.uint8</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.uint16.html">pyarrow.uint16</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.uint32.html">pyarrow.uint32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.uint64.html">pyarrow.uint64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.float16.html">pyarrow.float16</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.float32.html">pyarrow.float32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.float64.html">pyarrow.float64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.time32.html">pyarrow.time32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.time64.html">pyarrow.time64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.timestamp.html">pyarrow.timestamp</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.date32.html">pyarrow.date32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.date64.html">pyarrow.date64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.duration.html">pyarrow.duration</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.month_day_nano_interval.html">pyarrow.month_day_nano_interval</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.binary.html">pyarrow.binary</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.string.html">pyarrow.string</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.utf8.html">pyarrow.utf8</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.large_binary.html">pyarrow.large_binary</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.large_string.html">pyarrow.large_string</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.large_utf8.html">pyarrow.large_utf8</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.binary_view.html">pyarrow.binary_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.string_view.html">pyarrow.string_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.decimal128.html">pyarrow.decimal128</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.decimal256.html">pyarrow.decimal256</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.list_.html">pyarrow.list_</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.large_list.html">pyarrow.large_list</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.list_view.html">pyarrow.list_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.large_list_view.html">pyarrow.large_list_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.map_.html">pyarrow.map_</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.struct.html">pyarrow.struct</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dictionary.html">pyarrow.dictionary</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.run_end_encoded.html">pyarrow.run_end_encoded</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fixed_shape_tensor.html">pyarrow.fixed_shape_tensor</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.union.html">pyarrow.union</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dense_union.html">pyarrow.dense_union</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.sparse_union.html">pyarrow.sparse_union</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.opaque.html">pyarrow.opaque</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.bool8.html">pyarrow.bool8</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.uuid.html">pyarrow.uuid</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.json_.html">pyarrow.json_</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.field.html">pyarrow.field</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.schema.html">pyarrow.schema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.from_numpy_dtype.html">pyarrow.from_numpy_dtype</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.unify_schemas.html">pyarrow.unify_schemas</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DataType.html">pyarrow.DataType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DictionaryType.html">pyarrow.DictionaryType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ListType.html">pyarrow.ListType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ListViewType.html">pyarrow.ListViewType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedSizeListType.html">pyarrow.FixedSizeListType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeListType.html">pyarrow.LargeListType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeListViewType.html">pyarrow.LargeListViewType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.MapType.html">pyarrow.MapType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.StructType.html">pyarrow.StructType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UnionType.html">pyarrow.UnionType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DenseUnionType.html">pyarrow.DenseUnionType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.SparseUnionType.html">pyarrow.SparseUnionType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.TimestampType.html">pyarrow.TimestampType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Time32Type.html">pyarrow.Time32Type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Time64Type.html">pyarrow.Time64Type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DurationType.html">pyarrow.DurationType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedSizeBinaryType.html">pyarrow.FixedSizeBinaryType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal32Type.html">pyarrow.Decimal32Type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal64Type.html">pyarrow.Decimal64Type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal128Type.html">pyarrow.Decimal128Type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal256Type.html">pyarrow.Decimal256Type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Field.html">pyarrow.Field</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Schema.html">pyarrow.Schema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.RunEndEncodedType.html">pyarrow.RunEndEncodedType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BaseExtensionType.html">pyarrow.BaseExtensionType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ExtensionType.html">pyarrow.ExtensionType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UnknownExtensionType.html">pyarrow.UnknownExtensionType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.register_extension_type.html">pyarrow.register_extension_type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.unregister_extension_type.html">pyarrow.unregister_extension_type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedShapeTensorType.html">pyarrow.FixedShapeTensorType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.OpaqueType.html">pyarrow.OpaqueType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.JsonType.html">pyarrow.JsonType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UuidType.html">pyarrow.UuidType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Bool8Type.html">pyarrow.Bool8Type</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_boolean.html">pyarrow.types.is_boolean</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_integer.html">pyarrow.types.is_integer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_signed_integer.html">pyarrow.types.is_signed_integer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_unsigned_integer.html">pyarrow.types.is_unsigned_integer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_int8.html">pyarrow.types.is_int8</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_int16.html">pyarrow.types.is_int16</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_int32.html">pyarrow.types.is_int32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_int64.html">pyarrow.types.is_int64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_uint8.html">pyarrow.types.is_uint8</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_uint16.html">pyarrow.types.is_uint16</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_uint32.html">pyarrow.types.is_uint32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_uint64.html">pyarrow.types.is_uint64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_floating.html">pyarrow.types.is_floating</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_float16.html">pyarrow.types.is_float16</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_float32.html">pyarrow.types.is_float32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_float64.html">pyarrow.types.is_float64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_decimal.html">pyarrow.types.is_decimal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_decimal128.html">pyarrow.types.is_decimal128</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_decimal256.html">pyarrow.types.is_decimal256</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_list.html">pyarrow.types.is_list</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_large_list.html">pyarrow.types.is_large_list</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_fixed_size_list.html">pyarrow.types.is_fixed_size_list</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_list_view.html">pyarrow.types.is_list_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_large_list_view.html">pyarrow.types.is_large_list_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_struct.html">pyarrow.types.is_struct</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_union.html">pyarrow.types.is_union</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_nested.html">pyarrow.types.is_nested</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_run_end_encoded.html">pyarrow.types.is_run_end_encoded</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_temporal.html">pyarrow.types.is_temporal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_timestamp.html">pyarrow.types.is_timestamp</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_date.html">pyarrow.types.is_date</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_date32.html">pyarrow.types.is_date32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_date64.html">pyarrow.types.is_date64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_time.html">pyarrow.types.is_time</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_time32.html">pyarrow.types.is_time32</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_time64.html">pyarrow.types.is_time64</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_duration.html">pyarrow.types.is_duration</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_interval.html">pyarrow.types.is_interval</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_null.html">pyarrow.types.is_null</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_binary.html">pyarrow.types.is_binary</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_unicode.html">pyarrow.types.is_unicode</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_string.html">pyarrow.types.is_string</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_large_binary.html">pyarrow.types.is_large_binary</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_large_unicode.html">pyarrow.types.is_large_unicode</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_large_string.html">pyarrow.types.is_large_string</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_binary_view.html">pyarrow.types.is_binary_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_string_view.html">pyarrow.types.is_string_view</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_fixed_size_binary.html">pyarrow.types.is_fixed_size_binary</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_map.html">pyarrow.types.is_map</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_dictionary.html">pyarrow.types.is_dictionary</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.is_primitive.html">pyarrow.types.is_primitive</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.types.TypesEnum.html">pyarrow.types.TypesEnum</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/arrays.html">Arrays and Scalars</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.array.html">pyarrow.array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.nulls.html">pyarrow.nulls</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Array.html">pyarrow.Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BooleanArray.html">pyarrow.BooleanArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FloatingPointArray.html">pyarrow.FloatingPointArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.IntegerArray.html">pyarrow.IntegerArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int8Array.html">pyarrow.Int8Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int16Array.html">pyarrow.Int16Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int32Array.html">pyarrow.Int32Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int64Array.html">pyarrow.Int64Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.NullArray.html">pyarrow.NullArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.NumericArray.html">pyarrow.NumericArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt8Array.html">pyarrow.UInt8Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt16Array.html">pyarrow.UInt16Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt32Array.html">pyarrow.UInt32Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt64Array.html">pyarrow.UInt64Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.HalfFloatArray.html">pyarrow.HalfFloatArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FloatArray.html">pyarrow.FloatArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DoubleArray.html">pyarrow.DoubleArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BinaryArray.html">pyarrow.BinaryArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.StringArray.html">pyarrow.StringArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedSizeBinaryArray.html">pyarrow.FixedSizeBinaryArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeBinaryArray.html">pyarrow.LargeBinaryArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeStringArray.html">pyarrow.LargeStringArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BinaryViewArray.html">pyarrow.BinaryViewArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.StringViewArray.html">pyarrow.StringViewArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Time32Array.html">pyarrow.Time32Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Time64Array.html">pyarrow.Time64Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Date32Array.html">pyarrow.Date32Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Date64Array.html">pyarrow.Date64Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.TimestampArray.html">pyarrow.TimestampArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DurationArray.html">pyarrow.DurationArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.MonthDayNanoIntervalArray.html">pyarrow.MonthDayNanoIntervalArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal32Array.html">pyarrow.Decimal32Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal64Array.html">pyarrow.Decimal64Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal128Array.html">pyarrow.Decimal128Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal256Array.html">pyarrow.Decimal256Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DictionaryArray.html">pyarrow.DictionaryArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ListArray.html">pyarrow.ListArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedSizeListArray.html">pyarrow.FixedSizeListArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeListArray.html">pyarrow.LargeListArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ListViewArray.html">pyarrow.ListViewArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeListViewArray.html">pyarrow.LargeListViewArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.MapArray.html">pyarrow.MapArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.RunEndEncodedArray.html">pyarrow.RunEndEncodedArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.StructArray.html">pyarrow.StructArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UnionArray.html">pyarrow.UnionArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ExtensionArray.html">pyarrow.ExtensionArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedShapeTensorArray.html">pyarrow.FixedShapeTensorArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.OpaqueArray.html">pyarrow.OpaqueArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.JsonArray.html">pyarrow.JsonArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UuidArray.html">pyarrow.UuidArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Bool8Array.html">pyarrow.Bool8Array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.scalar.html">pyarrow.scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.NA.html">pyarrow.NA</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Scalar.html">pyarrow.Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BooleanScalar.html">pyarrow.BooleanScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int8Scalar.html">pyarrow.Int8Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int16Scalar.html">pyarrow.Int16Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int32Scalar.html">pyarrow.Int32Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Int64Scalar.html">pyarrow.Int64Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.NullScalar.html">pyarrow.NullScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt8Scalar.html">pyarrow.UInt8Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt16Scalar.html">pyarrow.UInt16Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt32Scalar.html">pyarrow.UInt32Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UInt64Scalar.html">pyarrow.UInt64Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.HalfFloatScalar.html">pyarrow.HalfFloatScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FloatScalar.html">pyarrow.FloatScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DoubleScalar.html">pyarrow.DoubleScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BinaryScalar.html">pyarrow.BinaryScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.StringScalar.html">pyarrow.StringScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedSizeBinaryScalar.html">pyarrow.FixedSizeBinaryScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeBinaryScalar.html">pyarrow.LargeBinaryScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeStringScalar.html">pyarrow.LargeStringScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BinaryViewScalar.html">pyarrow.BinaryViewScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.StringViewScalar.html">pyarrow.StringViewScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Time32Scalar.html">pyarrow.Time32Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Time64Scalar.html">pyarrow.Time64Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Date32Scalar.html">pyarrow.Date32Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Date64Scalar.html">pyarrow.Date64Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.TimestampScalar.html">pyarrow.TimestampScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DurationScalar.html">pyarrow.DurationScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.MonthDayNanoIntervalScalar.html">pyarrow.MonthDayNanoIntervalScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal128Scalar.html">pyarrow.Decimal128Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Decimal256Scalar.html">pyarrow.Decimal256Scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.DictionaryScalar.html">pyarrow.DictionaryScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.RunEndEncodedScalar.html">pyarrow.RunEndEncodedScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ListScalar.html">pyarrow.ListScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedSizeListScalar.html">pyarrow.FixedSizeListScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeListScalar.html">pyarrow.LargeListScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ListViewScalar.html">pyarrow.ListViewScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.LargeListViewScalar.html">pyarrow.LargeListViewScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.MapScalar.html">pyarrow.MapScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.StructScalar.html">pyarrow.StructScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UnionScalar.html">pyarrow.UnionScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ExtensionScalar.html">pyarrow.ExtensionScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedShapeTensorScalar.html">pyarrow.FixedShapeTensorScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.OpaqueScalar.html">pyarrow.OpaqueScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.JsonScalar.html">pyarrow.JsonScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.UuidScalar.html">pyarrow.UuidScalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Bool8Scalar.html">pyarrow.Bool8Scalar</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/memory.html">Buffers and Memory</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.allocate_buffer.html">pyarrow.allocate_buffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.py_buffer.html">pyarrow.py_buffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.foreign_buffer.html">pyarrow.foreign_buffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Buffer.html">pyarrow.Buffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ResizableBuffer.html">pyarrow.ResizableBuffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Codec.html">pyarrow.Codec</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compress.html">pyarrow.compress</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.decompress.html">pyarrow.decompress</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.MemoryPool.html">pyarrow.MemoryPool</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.default_memory_pool.html">pyarrow.default_memory_pool</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.jemalloc_memory_pool.html">pyarrow.jemalloc_memory_pool</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.mimalloc_memory_pool.html">pyarrow.mimalloc_memory_pool</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.system_memory_pool.html">pyarrow.system_memory_pool</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.jemalloc_set_decay_ms.html">pyarrow.jemalloc_set_decay_ms</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.set_memory_pool.html">pyarrow.set_memory_pool</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.log_memory_allocations.html">pyarrow.log_memory_allocations</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.total_allocated_bytes.html">pyarrow.total_allocated_bytes</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/tables.html">Tables and Tensors</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.chunked_array.html">pyarrow.chunked_array</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.concat_arrays.html">pyarrow.concat_arrays</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.concat_tables.html">pyarrow.concat_tables</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.record_batch.html">pyarrow.record_batch</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.concat_batches.html">pyarrow.concat_batches</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.table.html">pyarrow.table</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ChunkedArray.html">pyarrow.ChunkedArray</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.RecordBatch.html">pyarrow.RecordBatch</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Table.html">pyarrow.Table</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.TableGroupBy.html">pyarrow.TableGroupBy</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.RecordBatchReader.html">pyarrow.RecordBatchReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.interchange.from_dataframe.html">pyarrow.interchange.from_dataframe</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.Tensor.html">pyarrow.Tensor</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/compute.html">Compute Functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.all.html">pyarrow.compute.all</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.any.html">pyarrow.compute.any</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.approximate_median.html">pyarrow.compute.approximate_median</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.count.html">pyarrow.compute.count</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.count_distinct.html">pyarrow.compute.count_distinct</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.index.html">pyarrow.compute.index</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.max.html">pyarrow.compute.max</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.mean.html">pyarrow.compute.mean</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.min.html">pyarrow.compute.min</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.min_max.html">pyarrow.compute.min_max</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.mode.html">pyarrow.compute.mode</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.pivot_wider.html">pyarrow.compute.pivot_wider</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.product.html">pyarrow.compute.product</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.quantile.html">pyarrow.compute.quantile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.stddev.html">pyarrow.compute.stddev</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.sum.html">pyarrow.compute.sum</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.tdigest.html">pyarrow.compute.tdigest</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.variance.html">pyarrow.compute.variance</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cumulative_sum.html">pyarrow.compute.cumulative_sum</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cumulative_sum_checked.html">pyarrow.compute.cumulative_sum_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cumulative_prod.html">pyarrow.compute.cumulative_prod</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cumulative_prod_checked.html">pyarrow.compute.cumulative_prod_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cumulative_max.html">pyarrow.compute.cumulative_max</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cumulative_min.html">pyarrow.compute.cumulative_min</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.abs.html">pyarrow.compute.abs</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.abs_checked.html">pyarrow.compute.abs_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.add.html">pyarrow.compute.add</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.add_checked.html">pyarrow.compute.add_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.divide.html">pyarrow.compute.divide</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.divide_checked.html">pyarrow.compute.divide_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.multiply.html">pyarrow.compute.multiply</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.multiply_checked.html">pyarrow.compute.multiply_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.negate.html">pyarrow.compute.negate</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.negate_checked.html">pyarrow.compute.negate_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.power.html">pyarrow.compute.power</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.power_checked.html">pyarrow.compute.power_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.sign.html">pyarrow.compute.sign</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.sqrt.html">pyarrow.compute.sqrt</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.sqrt_checked.html">pyarrow.compute.sqrt_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.subtract.html">pyarrow.compute.subtract</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.subtract_checked.html">pyarrow.compute.subtract_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_and.html">pyarrow.compute.bit_wise_and</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_not.html">pyarrow.compute.bit_wise_not</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_or.html">pyarrow.compute.bit_wise_or</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.bit_wise_xor.html">pyarrow.compute.bit_wise_xor</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.shift_left.html">pyarrow.compute.shift_left</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.shift_left_checked.html">pyarrow.compute.shift_left_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.shift_right.html">pyarrow.compute.shift_right</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.shift_right_checked.html">pyarrow.compute.shift_right_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ceil.html">pyarrow.compute.ceil</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.floor.html">pyarrow.compute.floor</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.round.html">pyarrow.compute.round</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.round_to_multiple.html">pyarrow.compute.round_to_multiple</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.trunc.html">pyarrow.compute.trunc</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ln.html">pyarrow.compute.ln</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ln_checked.html">pyarrow.compute.ln_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.log10.html">pyarrow.compute.log10</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.log10_checked.html">pyarrow.compute.log10_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.log1p.html">pyarrow.compute.log1p</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.log1p_checked.html">pyarrow.compute.log1p_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.log2.html">pyarrow.compute.log2</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.log2_checked.html">pyarrow.compute.log2_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.logb.html">pyarrow.compute.logb</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.logb_checked.html">pyarrow.compute.logb_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.acos.html">pyarrow.compute.acos</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.acos_checked.html">pyarrow.compute.acos_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.asin.html">pyarrow.compute.asin</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.asin_checked.html">pyarrow.compute.asin_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.atan.html">pyarrow.compute.atan</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.atan2.html">pyarrow.compute.atan2</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cos.html">pyarrow.compute.cos</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cos_checked.html">pyarrow.compute.cos_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.sin.html">pyarrow.compute.sin</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.sin_checked.html">pyarrow.compute.sin_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.tan.html">pyarrow.compute.tan</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.tan_checked.html">pyarrow.compute.tan_checked</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.equal.html">pyarrow.compute.equal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.greater.html">pyarrow.compute.greater</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.greater_equal.html">pyarrow.compute.greater_equal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.less.html">pyarrow.compute.less</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.less_equal.html">pyarrow.compute.less_equal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.not_equal.html">pyarrow.compute.not_equal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.max_element_wise.html">pyarrow.compute.max_element_wise</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.min_element_wise.html">pyarrow.compute.min_element_wise</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.and_.html">pyarrow.compute.and_</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.and_kleene.html">pyarrow.compute.and_kleene</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.and_not.html">pyarrow.compute.and_not</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.and_not_kleene.html">pyarrow.compute.and_not_kleene</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.invert.html">pyarrow.compute.invert</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.or_.html">pyarrow.compute.or_</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.or_kleene.html">pyarrow.compute.or_kleene</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.xor.html">pyarrow.compute.xor</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_alnum.html">pyarrow.compute.ascii_is_alnum</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_alpha.html">pyarrow.compute.ascii_is_alpha</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_decimal.html">pyarrow.compute.ascii_is_decimal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_lower.html">pyarrow.compute.ascii_is_lower</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_printable.html">pyarrow.compute.ascii_is_printable</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_space.html">pyarrow.compute.ascii_is_space</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_upper.html">pyarrow.compute.ascii_is_upper</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_alnum.html">pyarrow.compute.utf8_is_alnum</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_alpha.html">pyarrow.compute.utf8_is_alpha</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_decimal.html">pyarrow.compute.utf8_is_decimal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_digit.html">pyarrow.compute.utf8_is_digit</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_lower.html">pyarrow.compute.utf8_is_lower</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_numeric.html">pyarrow.compute.utf8_is_numeric</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_printable.html">pyarrow.compute.utf8_is_printable</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_space.html">pyarrow.compute.utf8_is_space</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_upper.html">pyarrow.compute.utf8_is_upper</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_is_title.html">pyarrow.compute.ascii_is_title</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_is_title.html">pyarrow.compute.utf8_is_title</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.string_is_ascii.html">pyarrow.compute.string_is_ascii</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_capitalize.html">pyarrow.compute.ascii_capitalize</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_lower.html">pyarrow.compute.ascii_lower</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_reverse.html">pyarrow.compute.ascii_reverse</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_swapcase.html">pyarrow.compute.ascii_swapcase</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_title.html">pyarrow.compute.ascii_title</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_upper.html">pyarrow.compute.ascii_upper</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.binary_length.html">pyarrow.compute.binary_length</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.binary_repeat.html">pyarrow.compute.binary_repeat</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.binary_replace_slice.html">pyarrow.compute.binary_replace_slice</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.binary_reverse.html">pyarrow.compute.binary_reverse</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.replace_substring.html">pyarrow.compute.replace_substring</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.replace_substring_regex.html">pyarrow.compute.replace_substring_regex</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_capitalize.html">pyarrow.compute.utf8_capitalize</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_length.html">pyarrow.compute.utf8_length</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_lower.html">pyarrow.compute.utf8_lower</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_replace_slice.html">pyarrow.compute.utf8_replace_slice</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_reverse.html">pyarrow.compute.utf8_reverse</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_swapcase.html">pyarrow.compute.utf8_swapcase</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_title.html">pyarrow.compute.utf8_title</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_upper.html">pyarrow.compute.utf8_upper</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_center.html">pyarrow.compute.ascii_center</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_lpad.html">pyarrow.compute.ascii_lpad</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_rpad.html">pyarrow.compute.ascii_rpad</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_center.html">pyarrow.compute.utf8_center</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_lpad.html">pyarrow.compute.utf8_lpad</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_rpad.html">pyarrow.compute.utf8_rpad</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_zero_fill.html">pyarrow.compute.utf8_zero_fill</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_ltrim.html">pyarrow.compute.ascii_ltrim</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_ltrim_whitespace.html">pyarrow.compute.ascii_ltrim_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_rtrim.html">pyarrow.compute.ascii_rtrim</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_rtrim_whitespace.html">pyarrow.compute.ascii_rtrim_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_trim.html">pyarrow.compute.ascii_trim</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_trim_whitespace.html">pyarrow.compute.ascii_trim_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_ltrim.html">pyarrow.compute.utf8_ltrim</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_ltrim_whitespace.html">pyarrow.compute.utf8_ltrim_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_rtrim.html">pyarrow.compute.utf8_rtrim</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_rtrim_whitespace.html">pyarrow.compute.utf8_rtrim_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_trim.html">pyarrow.compute.utf8_trim</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_trim_whitespace.html">pyarrow.compute.utf8_trim_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ascii_split_whitespace.html">pyarrow.compute.ascii_split_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.split_pattern.html">pyarrow.compute.split_pattern</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.split_pattern_regex.html">pyarrow.compute.split_pattern_regex</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_split_whitespace.html">pyarrow.compute.utf8_split_whitespace</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.extract_regex.html">pyarrow.compute.extract_regex</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.binary_join.html">pyarrow.compute.binary_join</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.binary_join_element_wise.html">pyarrow.compute.binary_join_element_wise</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.binary_slice.html">pyarrow.compute.binary_slice</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.utf8_slice_codeunits.html">pyarrow.compute.utf8_slice_codeunits</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.count_substring.html">pyarrow.compute.count_substring</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.count_substring_regex.html">pyarrow.compute.count_substring_regex</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ends_with.html">pyarrow.compute.ends_with</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.find_substring.html">pyarrow.compute.find_substring</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.find_substring_regex.html">pyarrow.compute.find_substring_regex</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.index_in.html">pyarrow.compute.index_in</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.is_in.html">pyarrow.compute.is_in</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.match_like.html">pyarrow.compute.match_like</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.match_substring.html">pyarrow.compute.match_substring</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.match_substring_regex.html">pyarrow.compute.match_substring_regex</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.starts_with.html">pyarrow.compute.starts_with</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.indices_nonzero.html">pyarrow.compute.indices_nonzero</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.is_finite.html">pyarrow.compute.is_finite</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.is_inf.html">pyarrow.compute.is_inf</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.is_nan.html">pyarrow.compute.is_nan</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.is_null.html">pyarrow.compute.is_null</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.is_valid.html">pyarrow.compute.is_valid</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.true_unless_null.html">pyarrow.compute.true_unless_null</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.case_when.html">pyarrow.compute.case_when</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.choose.html">pyarrow.compute.choose</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.coalesce.html">pyarrow.compute.coalesce</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.if_else.html">pyarrow.compute.if_else</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.cast.html">pyarrow.compute.cast</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ceil_temporal.html">pyarrow.compute.ceil_temporal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.floor_temporal.html">pyarrow.compute.floor_temporal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.round_temporal.html">pyarrow.compute.round_temporal</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.run_end_decode.html">pyarrow.compute.run_end_decode</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.run_end_encode.html">pyarrow.compute.run_end_encode</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.strftime.html">pyarrow.compute.strftime</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.strptime.html">pyarrow.compute.strptime</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.day.html">pyarrow.compute.day</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.day_of_week.html">pyarrow.compute.day_of_week</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.day_of_year.html">pyarrow.compute.day_of_year</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.hour.html">pyarrow.compute.hour</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.iso_week.html">pyarrow.compute.iso_week</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.iso_year.html">pyarrow.compute.iso_year</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.iso_calendar.html">pyarrow.compute.iso_calendar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.is_leap_year.html">pyarrow.compute.is_leap_year</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.microsecond.html">pyarrow.compute.microsecond</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.millisecond.html">pyarrow.compute.millisecond</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.minute.html">pyarrow.compute.minute</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.month.html">pyarrow.compute.month</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.nanosecond.html">pyarrow.compute.nanosecond</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.quarter.html">pyarrow.compute.quarter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.second.html">pyarrow.compute.second</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.subsecond.html">pyarrow.compute.subsecond</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.us_week.html">pyarrow.compute.us_week</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.us_year.html">pyarrow.compute.us_year</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.week.html">pyarrow.compute.week</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.year.html">pyarrow.compute.year</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.year_month_day.html">pyarrow.compute.year_month_day</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.day_time_interval_between.html">pyarrow.compute.day_time_interval_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.days_between.html">pyarrow.compute.days_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.hours_between.html">pyarrow.compute.hours_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.microseconds_between.html">pyarrow.compute.microseconds_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.milliseconds_between.html">pyarrow.compute.milliseconds_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.minutes_between.html">pyarrow.compute.minutes_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.month_day_nano_interval_between.html">pyarrow.compute.month_day_nano_interval_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.month_interval_between.html">pyarrow.compute.month_interval_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.nanoseconds_between.html">pyarrow.compute.nanoseconds_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.quarters_between.html">pyarrow.compute.quarters_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.seconds_between.html">pyarrow.compute.seconds_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.weeks_between.html">pyarrow.compute.weeks_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.years_between.html">pyarrow.compute.years_between</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.assume_timezone.html">pyarrow.compute.assume_timezone</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.local_timestamp.html">pyarrow.compute.local_timestamp</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.dictionary_encode.html">pyarrow.compute.dictionary_encode</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.unique.html">pyarrow.compute.unique</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.value_counts.html">pyarrow.compute.value_counts</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.array_filter.html">pyarrow.compute.array_filter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.array_take.html">pyarrow.compute.array_take</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.drop_null.html">pyarrow.compute.drop_null</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.filter.html">pyarrow.compute.filter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.take.html">pyarrow.compute.take</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.array_sort_indices.html">pyarrow.compute.array_sort_indices</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.partition_nth_indices.html">pyarrow.compute.partition_nth_indices</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.select_k_unstable.html">pyarrow.compute.select_k_unstable</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.sort_indices.html">pyarrow.compute.sort_indices</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.fill_null.html">pyarrow.compute.fill_null</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.fill_null_backward.html">pyarrow.compute.fill_null_backward</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.fill_null_forward.html">pyarrow.compute.fill_null_forward</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.list_element.html">pyarrow.compute.list_element</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.list_flatten.html">pyarrow.compute.list_flatten</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.list_parent_indices.html">pyarrow.compute.list_parent_indices</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.list_slice.html">pyarrow.compute.list_slice</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.list_value_length.html">pyarrow.compute.list_value_length</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.make_struct.html">pyarrow.compute.make_struct</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.map_lookup.html">pyarrow.compute.map_lookup</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.replace_with_mask.html">pyarrow.compute.replace_with_mask</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.struct_field.html">pyarrow.compute.struct_field</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.pairwise_diff.html">pyarrow.compute.pairwise_diff</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ArraySortOptions.html">pyarrow.compute.ArraySortOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.AssumeTimezoneOptions.html">pyarrow.compute.AssumeTimezoneOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.CastOptions.html">pyarrow.compute.CastOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.CountOptions.html">pyarrow.compute.CountOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.CumulativeSumOptions.html">pyarrow.compute.CumulativeSumOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.DayOfWeekOptions.html">pyarrow.compute.DayOfWeekOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.DictionaryEncodeOptions.html">pyarrow.compute.DictionaryEncodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ElementWiseAggregateOptions.html">pyarrow.compute.ElementWiseAggregateOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ExtractRegexOptions.html">pyarrow.compute.ExtractRegexOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.FilterOptions.html">pyarrow.compute.FilterOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.IndexOptions.html">pyarrow.compute.IndexOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.JoinOptions.html">pyarrow.compute.JoinOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ListSliceOptions.html">pyarrow.compute.ListSliceOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.MakeStructOptions.html">pyarrow.compute.MakeStructOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.MapLookupOptions.html">pyarrow.compute.MapLookupOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.MatchSubstringOptions.html">pyarrow.compute.MatchSubstringOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ModeOptions.html">pyarrow.compute.ModeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.NullOptions.html">pyarrow.compute.NullOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.PadOptions.html">pyarrow.compute.PadOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ZeroFillOptions.html">pyarrow.compute.ZeroFillOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.PairwiseOptions.html">pyarrow.compute.PairwiseOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.PartitionNthOptions.html">pyarrow.compute.PartitionNthOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.PivotWiderOptions.html">pyarrow.compute.PivotWiderOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.QuantileOptions.html">pyarrow.compute.QuantileOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ReplaceSliceOptions.html">pyarrow.compute.ReplaceSliceOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ReplaceSubstringOptions.html">pyarrow.compute.ReplaceSubstringOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.RoundOptions.html">pyarrow.compute.RoundOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.RoundTemporalOptions.html">pyarrow.compute.RoundTemporalOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.RoundToMultipleOptions.html">pyarrow.compute.RoundToMultipleOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.RunEndEncodeOptions.html">pyarrow.compute.RunEndEncodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.ScalarAggregateOptions.html">pyarrow.compute.ScalarAggregateOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.SelectKOptions.html">pyarrow.compute.SelectKOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.SetLookupOptions.html">pyarrow.compute.SetLookupOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.SliceOptions.html">pyarrow.compute.SliceOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.SortOptions.html">pyarrow.compute.SortOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.SplitOptions.html">pyarrow.compute.SplitOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.SplitPatternOptions.html">pyarrow.compute.SplitPatternOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.StrftimeOptions.html">pyarrow.compute.StrftimeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.StrptimeOptions.html">pyarrow.compute.StrptimeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.StructFieldOptions.html">pyarrow.compute.StructFieldOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.TakeOptions.html">pyarrow.compute.TakeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.TDigestOptions.html">pyarrow.compute.TDigestOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.TrimOptions.html">pyarrow.compute.TrimOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.VarianceOptions.html">pyarrow.compute.VarianceOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.WeekOptions.html">pyarrow.compute.WeekOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.register_scalar_function.html">pyarrow.compute.register_scalar_function</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.UdfContext.html">pyarrow.compute.UdfContext</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.field.html">pyarrow.compute.field</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.compute.scalar.html">pyarrow.compute.scalar</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/acero.html">Acero - Streaming Execution Engine</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.Declaration.html">pyarrow.acero.Declaration</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.ExecNodeOptions.html">pyarrow.acero.ExecNodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.TableSourceNodeOptions.html">pyarrow.acero.TableSourceNodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.ScanNodeOptions.html">pyarrow.acero.ScanNodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.FilterNodeOptions.html">pyarrow.acero.FilterNodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.ProjectNodeOptions.html">pyarrow.acero.ProjectNodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.AggregateNodeOptions.html">pyarrow.acero.AggregateNodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.OrderByNodeOptions.html">pyarrow.acero.OrderByNodeOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.acero.HashJoinNodeOptions.html">pyarrow.acero.HashJoinNodeOptions</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/substrait.html">Substrait</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.run_query.html">pyarrow.substrait.run_query</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.BoundExpressions.html">pyarrow.substrait.BoundExpressions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.deserialize_expressions.html">pyarrow.substrait.deserialize_expressions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.serialize_expressions.html">pyarrow.substrait.serialize_expressions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.serialize_schema.html">pyarrow.substrait.serialize_schema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.deserialize_schema.html">pyarrow.substrait.deserialize_schema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.SubstraitSchema.html">pyarrow.substrait.SubstraitSchema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.substrait.get_supported_functions.html">pyarrow.substrait.get_supported_functions</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/files.html">Streams and File Access</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.input_stream.html">pyarrow.input_stream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.output_stream.html">pyarrow.output_stream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.memory_map.html">pyarrow.memory_map</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.create_memory_map.html">pyarrow.create_memory_map</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.NativeFile.html">pyarrow.NativeFile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.OSFile.html">pyarrow.OSFile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.PythonFile.html">pyarrow.PythonFile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BufferReader.html">pyarrow.BufferReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.BufferOutputStream.html">pyarrow.BufferOutputStream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.FixedSizeBufferWriter.html">pyarrow.FixedSizeBufferWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.MemoryMappedFile.html">pyarrow.MemoryMappedFile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.CompressedInputStream.html">pyarrow.CompressedInputStream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.CompressedOutputStream.html">pyarrow.CompressedOutputStream</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/ipc.html">Serialization and IPC</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.new_file.html">pyarrow.ipc.new_file</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.open_file.html">pyarrow.ipc.open_file</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.new_stream.html">pyarrow.ipc.new_stream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.open_stream.html">pyarrow.ipc.open_stream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.read_message.html">pyarrow.ipc.read_message</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.read_record_batch.html">pyarrow.ipc.read_record_batch</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.get_record_batch_size.html">pyarrow.ipc.get_record_batch_size</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.read_tensor.html">pyarrow.ipc.read_tensor</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.write_tensor.html">pyarrow.ipc.write_tensor</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.get_tensor_size.html">pyarrow.ipc.get_tensor_size</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.IpcReadOptions.html">pyarrow.ipc.IpcReadOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.IpcWriteOptions.html">pyarrow.ipc.IpcWriteOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.Message.html">pyarrow.ipc.Message</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.MessageReader.html">pyarrow.ipc.MessageReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchFileReader.html">pyarrow.ipc.RecordBatchFileReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchFileWriter.html">pyarrow.ipc.RecordBatchFileWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchStreamReader.html">pyarrow.ipc.RecordBatchStreamReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.ipc.RecordBatchStreamWriter.html">pyarrow.ipc.RecordBatchStreamWriter</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/flight.html">Arrow Flight</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.Action.html">pyarrow.flight.Action</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ActionType.html">pyarrow.flight.ActionType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.DescriptorType.html">pyarrow.flight.DescriptorType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightDescriptor.html">pyarrow.flight.FlightDescriptor</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightEndpoint.html">pyarrow.flight.FlightEndpoint</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightInfo.html">pyarrow.flight.FlightInfo</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.Location.html">pyarrow.flight.Location</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.MetadataRecordBatchReader.html">pyarrow.flight.MetadataRecordBatchReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.MetadataRecordBatchWriter.html">pyarrow.flight.MetadataRecordBatchWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.Ticket.html">pyarrow.flight.Ticket</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.Result.html">pyarrow.flight.Result</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.connect.html">pyarrow.flight.connect</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightCallOptions.html">pyarrow.flight.FlightCallOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightClient.html">pyarrow.flight.FlightClient</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightStreamReader.html">pyarrow.flight.FlightStreamReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightStreamWriter.html">pyarrow.flight.FlightStreamWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ClientMiddlewareFactory.html">pyarrow.flight.ClientMiddlewareFactory</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ClientMiddleware.html">pyarrow.flight.ClientMiddleware</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightDataStream.html">pyarrow.flight.FlightDataStream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightMetadataWriter.html">pyarrow.flight.FlightMetadataWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightServerBase.html">pyarrow.flight.FlightServerBase</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.GeneratorStream.html">pyarrow.flight.GeneratorStream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.RecordBatchStream.html">pyarrow.flight.RecordBatchStream</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ServerCallContext.html">pyarrow.flight.ServerCallContext</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ServerMiddlewareFactory.html">pyarrow.flight.ServerMiddlewareFactory</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ServerMiddleware.html">pyarrow.flight.ServerMiddleware</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ClientAuthHandler.html">pyarrow.flight.ClientAuthHandler</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.ServerAuthHandler.html">pyarrow.flight.ServerAuthHandler</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightError.html">pyarrow.flight.FlightError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightCancelledError.html">pyarrow.flight.FlightCancelledError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightInternalError.html">pyarrow.flight.FlightInternalError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightServerError.html">pyarrow.flight.FlightServerError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightTimedOutError.html">pyarrow.flight.FlightTimedOutError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightUnauthenticatedError.html">pyarrow.flight.FlightUnauthenticatedError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightUnauthorizedError.html">pyarrow.flight.FlightUnauthorizedError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightUnavailableError.html">pyarrow.flight.FlightUnavailableError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightWriteSizeExceededError.html">pyarrow.flight.FlightWriteSizeExceededError</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.FlightMethod.html">pyarrow.flight.FlightMethod</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.flight.CallInfo.html">pyarrow.flight.CallInfo</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/formats.html">Tabular File Formats</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.ConvertOptions.html">pyarrow.csv.ConvertOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.CSVStreamingReader.html">pyarrow.csv.CSVStreamingReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.CSVWriter.html">pyarrow.csv.CSVWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.ISO8601.html">pyarrow.csv.ISO8601</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.ParseOptions.html">pyarrow.csv.ParseOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.ReadOptions.html">pyarrow.csv.ReadOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.WriteOptions.html">pyarrow.csv.WriteOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.open_csv.html">pyarrow.csv.open_csv</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.read_csv.html">pyarrow.csv.read_csv</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.write_csv.html">pyarrow.csv.write_csv</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.csv.InvalidRow.html">pyarrow.csv.InvalidRow</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.feather.read_feather.html">pyarrow.feather.read_feather</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.feather.read_table.html">pyarrow.feather.read_table</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.feather.write_feather.html">pyarrow.feather.write_feather</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.json.ReadOptions.html">pyarrow.json.ReadOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.json.ParseOptions.html">pyarrow.json.ParseOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.json.open_json.html">pyarrow.json.open_json</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.json.read_json.html">pyarrow.json.read_json</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.ParquetDataset.html">pyarrow.parquet.ParquetDataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.ParquetFile.html">pyarrow.parquet.ParquetFile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.ParquetWriter.html">pyarrow.parquet.ParquetWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.read_table.html">pyarrow.parquet.read_table</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.read_metadata.html">pyarrow.parquet.read_metadata</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.read_pandas.html">pyarrow.parquet.read_pandas</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.read_schema.html">pyarrow.parquet.read_schema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.write_metadata.html">pyarrow.parquet.write_metadata</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.write_table.html">pyarrow.parquet.write_table</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.write_to_dataset.html">pyarrow.parquet.write_to_dataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.FileMetaData.html">pyarrow.parquet.FileMetaData</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.RowGroupMetaData.html">pyarrow.parquet.RowGroupMetaData</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.SortingColumn.html">pyarrow.parquet.SortingColumn</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.ColumnChunkMetaData.html">pyarrow.parquet.ColumnChunkMetaData</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.Statistics.html">pyarrow.parquet.Statistics</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.ParquetSchema.html">pyarrow.parquet.ParquetSchema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.ColumnSchema.html">pyarrow.parquet.ColumnSchema</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.ParquetLogicalType.html">pyarrow.parquet.ParquetLogicalType</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.encryption.CryptoFactory.html">pyarrow.parquet.encryption.CryptoFactory</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.encryption.KmsClient.html">pyarrow.parquet.encryption.KmsClient</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.encryption.KmsConnectionConfig.html">pyarrow.parquet.encryption.KmsConnectionConfig</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.encryption.EncryptionConfiguration.html">pyarrow.parquet.encryption.EncryptionConfiguration</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.parquet.encryption.DecryptionConfiguration.html">pyarrow.parquet.encryption.DecryptionConfiguration</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.orc.ORCFile.html">pyarrow.orc.ORCFile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.orc.ORCWriter.html">pyarrow.orc.ORCWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.orc.read_table.html">pyarrow.orc.read_table</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.orc.write_table.html">pyarrow.orc.write_table</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/filesystems.html">Filesystems</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.FileInfo.html">pyarrow.fs.FileInfo</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.FileSelector.html">pyarrow.fs.FileSelector</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.FileSystem.html">pyarrow.fs.FileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.LocalFileSystem.html">pyarrow.fs.LocalFileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.S3FileSystem.html">pyarrow.fs.S3FileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.GcsFileSystem.html">pyarrow.fs.GcsFileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.HadoopFileSystem.html">pyarrow.fs.HadoopFileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.SubTreeFileSystem.html">pyarrow.fs.SubTreeFileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.AzureFileSystem.html">pyarrow.fs.AzureFileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.PyFileSystem.html">pyarrow.fs.PyFileSystem</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.FileSystemHandler.html">pyarrow.fs.FileSystemHandler</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.FSSpecHandler.html">pyarrow.fs.FSSpecHandler</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.copy_files.html">pyarrow.fs.copy_files</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.initialize_s3.html">pyarrow.fs.initialize_s3</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.finalize_s3.html">pyarrow.fs.finalize_s3</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.resolve_s3_region.html">pyarrow.fs.resolve_s3_region</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.fs.S3LogLevel.html">pyarrow.fs.S3LogLevel</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/dataset.html">Dataset</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.dataset.html">pyarrow.dataset.dataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.parquet_dataset.html">pyarrow.dataset.parquet_dataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.partitioning.html">pyarrow.dataset.partitioning</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.field.html">pyarrow.dataset.field</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.scalar.html">pyarrow.dataset.scalar</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.write_dataset.html">pyarrow.dataset.write_dataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.FileFormat.html">pyarrow.dataset.FileFormat</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.CsvFileFormat.html">pyarrow.dataset.CsvFileFormat</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.CsvFragmentScanOptions.html">pyarrow.dataset.CsvFragmentScanOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.IpcFileFormat.html">pyarrow.dataset.IpcFileFormat</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.JsonFileFormat.html">pyarrow.dataset.JsonFileFormat</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.ParquetFileFormat.html">pyarrow.dataset.ParquetFileFormat</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.ParquetReadOptions.html">pyarrow.dataset.ParquetReadOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.ParquetFragmentScanOptions.html">pyarrow.dataset.ParquetFragmentScanOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.ParquetFileFragment.html">pyarrow.dataset.ParquetFileFragment</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.OrcFileFormat.html">pyarrow.dataset.OrcFileFormat</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.Partitioning.html">pyarrow.dataset.Partitioning</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.PartitioningFactory.html">pyarrow.dataset.PartitioningFactory</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.DirectoryPartitioning.html">pyarrow.dataset.DirectoryPartitioning</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.HivePartitioning.html">pyarrow.dataset.HivePartitioning</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.FilenamePartitioning.html">pyarrow.dataset.FilenamePartitioning</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.Dataset.html">pyarrow.dataset.Dataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.FileSystemDataset.html">pyarrow.dataset.FileSystemDataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.FileSystemFactoryOptions.html">pyarrow.dataset.FileSystemFactoryOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.FileSystemDatasetFactory.html">pyarrow.dataset.FileSystemDatasetFactory</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.UnionDataset.html">pyarrow.dataset.UnionDataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.Fragment.html">pyarrow.dataset.Fragment</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.FragmentScanOptions.html">pyarrow.dataset.FragmentScanOptions</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.TaggedRecordBatch.html">pyarrow.dataset.TaggedRecordBatch</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.Scanner.html">pyarrow.dataset.Scanner</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.Expression.html">pyarrow.dataset.Expression</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.InMemoryDataset.html">pyarrow.dataset.InMemoryDataset</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.WrittenFile.html">pyarrow.dataset.WrittenFile</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.dataset.get_partition_keys.html">pyarrow.dataset.get_partition_keys</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/cuda.html">CUDA Integration</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.Context.html">pyarrow.cuda.Context</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.CudaBuffer.html">pyarrow.cuda.CudaBuffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.new_host_buffer.html">pyarrow.cuda.new_host_buffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.HostBuffer.html">pyarrow.cuda.HostBuffer</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.BufferReader.html">pyarrow.cuda.BufferReader</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.BufferWriter.html">pyarrow.cuda.BufferWriter</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.serialize_record_batch.html">pyarrow.cuda.serialize_record_batch</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.read_record_batch.html">pyarrow.cuda.read_record_batch</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.read_message.html">pyarrow.cuda.read_message</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cuda.IpcMemHandle.html">pyarrow.cuda.IpcMemHandle</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l3 has-children"><a class="reference internal" href="api/misc.html">Miscellaneous</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.cpu_count.html">pyarrow.cpu_count</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.set_cpu_count.html">pyarrow.set_cpu_count</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.io_thread_count.html">pyarrow.io_thread_count</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.set_io_thread_count.html">pyarrow.set_io_thread_count</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.get_include.html">pyarrow.get_include</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.get_libraries.html">pyarrow.get_libraries</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="generated/pyarrow.get_library_dirs.html">pyarrow.get_library_dirs</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="getting_involved.html">Getting Involved</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="benchmarks.html">Benchmarks</a></li> |
| <li class="toctree-l2"><a class="reference external" href="https://arrow.apache.org/cookbook/py/">Python cookbook</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/docs/r">R</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/main/ruby/README.md">Ruby</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://docs.rs/crate/arrow/">Rust</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow-swift/blob/main/Arrow/README.md">Swift</a></li> |
| <li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/nanoarrow/">nanoarrow</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../status.html">Implementation Status</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../implementations.html" class="nav-link">Implementations</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Python</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Data Types and In-Memory Data Model</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="data-types-and-in-memory-data-model"> |
| <span id="data"></span><h1>Data Types and In-Memory Data Model<a class="headerlink" href="#data-types-and-in-memory-data-model" title="Link to this heading">#</a></h1> |
| <p>Apache Arrow defines columnar array data structures by composing type metadata |
| with memory buffers, like the ones explained in the documentation on |
| <a class="reference internal" href="memory.html#io"><span class="std std-ref">Memory and IO</span></a>. These data structures are exposed in Python through |
| a series of interrelated classes:</p> |
| <ul class="simple"> |
| <li><p><strong>Type Metadata</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.DataType</span></code>, which describe the |
| type of an array and govern how its values are interpreted</p></li> |
| <li><p><strong>Schemas</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.Schema</span></code>, which describe a named |
| collection of types. These can be thought of as the column types in a |
| table-like object.</p></li> |
| <li><p><strong>Arrays</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.Array</span></code>, which are atomic, contiguous |
| columnar data structures composed from Arrow Buffer objects</p></li> |
| <li><p><strong>Record Batches</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.RecordBatch</span></code>, which are a |
| collection of Array objects with a particular Schema</p></li> |
| <li><p><strong>Tables</strong>: Instances of <code class="docutils literal notranslate"><span class="pre">pyarrow.Table</span></code>, a logical table data structure in |
| which each column consists of one or more <code class="docutils literal notranslate"><span class="pre">pyarrow.Array</span></code> objects of the |
| same type.</p></li> |
| </ul> |
| <p>We will examine these in the sections below in a series of examples.</p> |
| <section id="type-metadata"> |
| <span id="data-types"></span><h2>Type Metadata<a class="headerlink" href="#type-metadata" title="Link to this heading">#</a></h2> |
| <p>Apache Arrow defines language agnostic column-oriented data structures for |
| array data. These include:</p> |
| <ul class="simple"> |
| <li><p><strong>Fixed-length primitive types</strong>: numbers, booleans, date and times, fixed |
| size binary, decimals, and other values that fit into a given number</p></li> |
| <li><p><strong>Variable-length primitive types</strong>: binary, string</p></li> |
| <li><p><strong>Nested types</strong>: list, map, struct, and union</p></li> |
| <li><p><strong>Dictionary type</strong>: An encoded categorical type (more on this later)</p></li> |
| </ul> |
| <p>Each data type in Arrow has a corresponding factory function for creating |
| an instance of that type object in Python:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [1]: </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| |
| <span class="gp">In [2]: </span><span class="n">t1</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">()</span> |
| |
| <span class="gp">In [3]: </span><span class="n">t2</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">()</span> |
| |
| <span class="gp">In [4]: </span><span class="n">t3</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">binary</span><span class="p">()</span> |
| |
| <span class="gp">In [5]: </span><span class="n">t4</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">binary</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> |
| |
| <span class="gp">In [6]: </span><span class="n">t5</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'ms'</span><span class="p">)</span> |
| |
| <span class="gp">In [7]: </span><span class="n">t1</span> |
| <span class="gh">Out[7]: </span><span class="go">DataType(int32)</span> |
| |
| <span class="gp">In [8]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span> |
| <span class="go">int32</span> |
| |
| <span class="gp">In [9]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t4</span><span class="p">)</span> |
| <span class="go">fixed_size_binary[10]</span> |
| |
| <span class="gp">In [10]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t5</span><span class="p">)</span> |
| <span class="go">timestamp[ms]</span> |
| </pre></div> |
| </div> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Different data types might use a given physical storage. For example, |
| <code class="docutils literal notranslate"><span class="pre">int64</span></code>, <code class="docutils literal notranslate"><span class="pre">float64</span></code>, and <code class="docutils literal notranslate"><span class="pre">timestamp[ms]</span></code> all occupy 64 bits per value.</p> |
| </div> |
| <p>These objects are <code class="docutils literal notranslate"><span class="pre">metadata</span></code>; they are used for describing the data in arrays, |
| schemas, and record batches. In Python, they can be used in functions where the |
| input data (e.g. Python objects) may be coerced to more than one Arrow type.</p> |
| <p>The <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field" title="pyarrow.Field"><code class="xref py py-class docutils literal notranslate"><span class="pre">Field</span></code></a> type is a type plus a name and optional |
| user-defined metadata:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [11]: </span><span class="n">f0</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'int32_field'</span><span class="p">,</span> <span class="n">t1</span><span class="p">)</span> |
| |
| <span class="gp">In [12]: </span><span class="n">f0</span> |
| <span class="gh">Out[12]: </span><span class="go">pyarrow.Field<int32_field: int32></span> |
| |
| <span class="gp">In [13]: </span><span class="n">f0</span><span class="o">.</span><span class="n">name</span> |
| <span class="gh">Out[13]: </span><span class="go">'int32_field'</span> |
| |
| <span class="gp">In [14]: </span><span class="n">f0</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[14]: </span><span class="go">DataType(int32)</span> |
| </pre></div> |
| </div> |
| <p>Arrow supports <strong>nested value types</strong> like list, map, struct, and union. When |
| creating these, you must pass types or fields to indicate the data types of the |
| types’ children. For example, we can define a list of int32 values with:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [15]: </span><span class="n">t6</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">list_</span><span class="p">(</span><span class="n">t1</span><span class="p">)</span> |
| |
| <span class="gp">In [16]: </span><span class="n">t6</span> |
| <span class="gh">Out[16]: </span><span class="go">ListType(list<item: int32>)</span> |
| </pre></div> |
| </div> |
| <p>A <code class="docutils literal notranslate"><span class="pre">struct</span></code> is a collection of named fields:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [17]: </span><span class="n">fields</span> <span class="o">=</span> <span class="p">[</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s0'</span><span class="p">,</span> <span class="n">t1</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s1'</span><span class="p">,</span> <span class="n">t2</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s2'</span><span class="p">,</span> <span class="n">t4</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'s3'</span><span class="p">,</span> <span class="n">t6</span><span class="p">),</span> |
| <span class="gp"> ....: </span><span class="p">]</span> |
| <span class="gp"> ....: </span> |
| |
| <span class="gp">In [18]: </span><span class="n">t7</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">(</span><span class="n">fields</span><span class="p">)</span> |
| |
| <span class="gp">In [19]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t7</span><span class="p">)</span> |
| <span class="go">struct<s0: int32, s1: string, s2: fixed_size_binary[10], s3: list<item: int32>></span> |
| </pre></div> |
| </div> |
| <p>For convenience, you can pass <code class="docutils literal notranslate"><span class="pre">(name,</span> <span class="pre">type)</span></code> tuples directly instead of |
| <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field" title="pyarrow.Field"><code class="xref py py-class docutils literal notranslate"><span class="pre">Field</span></code></a> instances:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [20]: </span><span class="n">t8</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">([(</span><span class="s1">'s0'</span><span class="p">,</span> <span class="n">t1</span><span class="p">),</span> <span class="p">(</span><span class="s1">'s1'</span><span class="p">,</span> <span class="n">t2</span><span class="p">),</span> <span class="p">(</span><span class="s1">'s2'</span><span class="p">,</span> <span class="n">t4</span><span class="p">),</span> <span class="p">(</span><span class="s1">'s3'</span><span class="p">,</span> <span class="n">t6</span><span class="p">)])</span> |
| |
| <span class="gp">In [21]: </span><span class="nb">print</span><span class="p">(</span><span class="n">t8</span><span class="p">)</span> |
| <span class="go">struct<s0: int32, s1: string, s2: fixed_size_binary[10], s3: list<item: int32>></span> |
| |
| <span class="gp">In [22]: </span><span class="n">t8</span> <span class="o">==</span> <span class="n">t7</span> |
| <span class="gh">Out[22]: </span><span class="go">True</span> |
| </pre></div> |
| </div> |
| <p>See <a class="reference internal" href="api/datatypes.html#api-types"><span class="std std-ref">Data Types API</span></a> for a full listing of data type |
| functions.</p> |
| </section> |
| <section id="schemas"> |
| <span id="data-schema"></span><h2>Schemas<a class="headerlink" href="#schemas" title="Link to this heading">#</a></h2> |
| <p>The <a class="reference internal" href="generated/pyarrow.Schema.html#pyarrow.Schema" title="pyarrow.Schema"><code class="xref py py-class docutils literal notranslate"><span class="pre">Schema</span></code></a> type is similar to the <code class="docutils literal notranslate"><span class="pre">struct</span></code> array type; it |
| defines the column names and types in a record batch or table data |
| structure. The <a class="reference internal" href="generated/pyarrow.schema.html#pyarrow.schema" title="pyarrow.schema"><code class="xref py py-func docutils literal notranslate"><span class="pre">pyarrow.schema()</span></code></a> factory function makes new Schema objects in |
| Python:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [23]: </span><span class="n">my_schema</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([(</span><span class="s1">'field0'</span><span class="p">,</span> <span class="n">t1</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'field1'</span><span class="p">,</span> <span class="n">t2</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'field2'</span><span class="p">,</span> <span class="n">t4</span><span class="p">),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'field3'</span><span class="p">,</span> <span class="n">t6</span><span class="p">)])</span> |
| <span class="gp"> ....: </span> |
| |
| <span class="gp">In [24]: </span><span class="n">my_schema</span> |
| <span class="gh">Out[24]: </span> |
| <span class="go">field0: int32</span> |
| <span class="go">field1: string</span> |
| <span class="go">field2: fixed_size_binary[10]</span> |
| <span class="go">field3: list<item: int32></span> |
| <span class="go"> child 0, item: int32</span> |
| </pre></div> |
| </div> |
| <p>In some applications, you may not create schemas directly, only using the ones |
| that are embedded in <a class="reference internal" href="ipc.html#ipc"><span class="std std-ref">IPC messages</span></a>.</p> |
| <p>Schemas are immutable, which means you can’t update an existing schema, but you |
| can create a new one with updated values using <a class="reference internal" href="generated/pyarrow.Schema.html#pyarrow.Schema.set" title="pyarrow.Schema.set"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Schema.set()</span></code></a>.</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [25]: </span><span class="n">updated_field</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'field0_new'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">())</span> |
| |
| <span class="gp">In [26]: </span><span class="n">my_schema2</span> <span class="o">=</span> <span class="n">my_schema</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">updated_field</span><span class="p">)</span> |
| |
| <span class="gp">In [27]: </span><span class="n">my_schema2</span> |
| <span class="gh">Out[27]: </span> |
| <span class="go">field0_new: int64</span> |
| <span class="go">field1: string</span> |
| <span class="go">field2: fixed_size_binary[10]</span> |
| <span class="go">field3: list<item: int32></span> |
| <span class="go"> child 0, item: int32</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="arrays"> |
| <span id="data-array"></span><h2>Arrays<a class="headerlink" href="#arrays" title="Link to this heading">#</a></h2> |
| <p>For each data type, there is an accompanying array data structure for holding |
| memory buffers that define a single contiguous chunk of columnar array |
| data. When you are using PyArrow, this data may come from IPC tools, though it |
| can also be created from various types of Python sequences (lists, NumPy |
| arrays, pandas data).</p> |
| <p>A simple way to create arrays is with <code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code>, which is similar to |
| the <code class="docutils literal notranslate"><span class="pre">numpy.array</span></code> function. By default PyArrow will infer the data type |
| for you:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [28]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span> |
| |
| <span class="gp">In [29]: </span><span class="n">arr</span> |
| <span class="gh">Out[29]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7feff0647400></span> |
| <span class="go">[</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> null,</span> |
| <span class="go"> 3</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>But you may also pass a specific data type to override type inference:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [30]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">uint16</span><span class="p">())</span> |
| <span class="gh">Out[30]: </span> |
| <span class="go"><pyarrow.lib.UInt16Array object at 0x7feff06479a0></span> |
| <span class="go">[</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>The array’s <code class="docutils literal notranslate"><span class="pre">type</span></code> attribute is the corresponding piece of type metadata:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [31]: </span><span class="n">arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[31]: </span><span class="go">DataType(int64)</span> |
| </pre></div> |
| </div> |
| <p>Each in-memory array has a known length and null count (which will be 0 if |
| there are no null values):</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [32]: </span><span class="nb">len</span><span class="p">(</span><span class="n">arr</span><span class="p">)</span> |
| <span class="gh">Out[32]: </span><span class="go">4</span> |
| |
| <span class="gp">In [33]: </span><span class="n">arr</span><span class="o">.</span><span class="n">null_count</span> |
| <span class="gh">Out[33]: </span><span class="go">1</span> |
| </pre></div> |
| </div> |
| <p>Scalar values can be selected with normal indexing. <code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> converts |
| <code class="docutils literal notranslate"><span class="pre">None</span></code> values to Arrow nulls; we return the special <code class="docutils literal notranslate"><span class="pre">pyarrow.NA</span></code> value for |
| nulls:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [34]: </span><span class="n">arr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> |
| <span class="gh">Out[34]: </span><span class="go"><pyarrow.Int64Scalar: 1></span> |
| |
| <span class="gp">In [35]: </span><span class="n">arr</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> |
| <span class="gh">Out[35]: </span><span class="go"><pyarrow.Int64Scalar: None></span> |
| </pre></div> |
| </div> |
| <p>Arrow data is immutable, so values can be selected but not assigned.</p> |
| <p>Arrays can be sliced without copying:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [36]: </span><span class="n">arr</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="mi">3</span><span class="p">]</span> |
| <span class="gh">Out[36]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7feff05a0460></span> |
| <span class="go">[</span> |
| <span class="go"> 2,</span> |
| <span class="go"> null</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <section id="none-values-and-nan-handling"> |
| <h3>None values and NAN handling<a class="headerlink" href="#none-values-and-nan-handling" title="Link to this heading">#</a></h3> |
| <p>As mentioned in the above section, the Python object <code class="docutils literal notranslate"><span class="pre">None</span></code> is always |
| converted to an Arrow null element on the conversion to <code class="docutils literal notranslate"><span class="pre">pyarrow.Array</span></code>. For |
| the float NaN value which is either represented by the Python object |
| <code class="docutils literal notranslate"><span class="pre">float('nan')</span></code> or <code class="docutils literal notranslate"><span class="pre">numpy.nan</span></code> we normally convert it to a <em>valid</em> float |
| value during the conversion. If an integer input is supplied to |
| <code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> that contains <code class="docutils literal notranslate"><span class="pre">np.nan</span></code>, <code class="docutils literal notranslate"><span class="pre">ValueError</span></code> is raised.</p> |
| <p>To handle better compatibility with Pandas, we support interpreting NaN values as |
| null elements. This is enabled automatically on all <code class="docutils literal notranslate"><span class="pre">from_pandas</span></code> function and |
| can be enabled on the other conversion functions by passing <code class="docutils literal notranslate"><span class="pre">from_pandas=True</span></code> |
| as a function parameter.</p> |
| </section> |
| <section id="list-arrays"> |
| <h3>List arrays<a class="headerlink" href="#list-arrays" title="Link to this heading">#</a></h3> |
| <p><code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> is able to infer the type of simple nested data structures |
| like lists:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [37]: </span><span class="n">nested_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([[],</span> <span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">]])</span> |
| |
| <span class="gp">In [38]: </span><span class="nb">print</span><span class="p">(</span><span class="n">nested_arr</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> |
| <span class="go">list<item: int64></span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="listview-arrays"> |
| <h3>ListView arrays<a class="headerlink" href="#listview-arrays" title="Link to this heading">#</a></h3> |
| <p><code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> can create an alternate list type called ListView:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [39]: </span><span class="n">nested_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([[],</span> <span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">]],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">list_view</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">()))</span> |
| |
| <span class="gp">In [40]: </span><span class="nb">print</span><span class="p">(</span><span class="n">nested_arr</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> |
| <span class="go">list_view<item: int64></span> |
| </pre></div> |
| </div> |
| <p>ListView arrays have a different set of buffers than List arrays. The ListView array |
| has both an offsets and sizes buffer, while a List array only has an offsets buffer. |
| This allows for ListView arrays to specify out-of-order offsets:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [41]: </span><span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">]</span> |
| |
| <span class="gp">In [42]: </span><span class="n">offsets</span> <span class="o">=</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span> |
| |
| <span class="gp">In [43]: </span><span class="n">sizes</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span> |
| |
| <span class="gp">In [44]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">ListViewArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">offsets</span><span class="p">,</span> <span class="n">sizes</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span> |
| |
| <span class="gp">In [45]: </span><span class="n">arr</span> |
| <span class="gh">Out[45]: </span> |
| <span class="go"><pyarrow.lib.ListViewArray object at 0x7feff05a0f40></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>See the format specification for more details on <a class="reference internal" href="../format/Columnar.html#listview-layout"><span class="std std-ref">ListView Layout</span></a>.</p> |
| </section> |
| <section id="struct-arrays"> |
| <h3>Struct arrays<a class="headerlink" href="#struct-arrays" title="Link to this heading">#</a></h3> |
| <p><code class="docutils literal notranslate"><span class="pre">pyarrow.array</span></code> is able to infer the schema of a struct type from arrays of |
| dictionaries:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [46]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span> <span class="p">{</span><span class="s1">'z'</span><span class="p">:</span> <span class="mf">3.4</span><span class="p">,</span> <span class="s1">'x'</span><span class="p">:</span> <span class="mi">4</span><span class="p">}])</span> |
| <span class="gh">Out[46]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7feff05a1420></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int64</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> null</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 2 type: double</span> |
| <span class="go"> [</span> |
| <span class="go"> null,</span> |
| <span class="go"> 3.4</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>Struct arrays can be initialized from a sequence of Python dicts or tuples. For tuples, |
| you must explicitly pass the type:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [47]: </span><span class="n">ty</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">([(</span><span class="s1">'x'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">()),</span> |
| <span class="gp"> ....: </span> <span class="p">(</span><span class="s1">'y'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">())])</span> |
| <span class="gp"> ....: </span> |
| |
| <span class="gp">In [48]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span> <span class="p">{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">:</span> <span class="kc">False</span><span class="p">}],</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[48]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7feff05a1ba0></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int8</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> false</span> |
| <span class="go"> ]</span> |
| |
| <span class="gp">In [49]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([(</span><span class="mi">3</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span> <span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="kc">False</span><span class="p">)],</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[49]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7feff05a1cc0></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int8</span> |
| <span class="go"> [</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> false</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>When initializing a struct array, nulls are allowed both at the struct |
| level and at the individual field level. If initializing from a sequence |
| of Python dicts, a missing dict key is handled as a null value:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [50]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([{</span><span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="kc">None</span><span class="p">,</span> <span class="p">{</span><span class="s1">'y'</span><span class="p">:</span> <span class="kc">None</span><span class="p">}],</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[50]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7feff05a1d80></span> |
| <span class="go">-- is_valid:</span> |
| <span class="go"> [</span> |
| <span class="go"> true,</span> |
| <span class="go"> false,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 0 type: int8</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> null</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> null,</span> |
| <span class="go"> false,</span> |
| <span class="go"> null</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>You can also construct a struct array from existing arrays for each of the |
| struct’s components. In this case, data storage will be shared with the |
| individual arrays, and no copy is involved:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [51]: </span><span class="n">xs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int16</span><span class="p">())</span> |
| |
| <span class="gp">In [52]: </span><span class="n">ys</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| |
| <span class="gp">In [53]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">StructArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">((</span><span class="n">xs</span><span class="p">,</span> <span class="n">ys</span><span class="p">),</span> <span class="n">names</span><span class="o">=</span><span class="p">(</span><span class="s1">'x'</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">))</span> |
| |
| <span class="gp">In [54]: </span><span class="n">arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[54]: </span><span class="go">StructType(struct<x: int16, y: bool>)</span> |
| |
| <span class="gp">In [55]: </span><span class="n">arr</span> |
| <span class="gh">Out[55]: </span> |
| <span class="go"><pyarrow.lib.StructArray object at 0x7feff05a2620></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- child 0 type: int16</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6,</span> |
| <span class="go"> 7</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> false,</span> |
| <span class="go"> true,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="map-arrays"> |
| <h3>Map arrays<a class="headerlink" href="#map-arrays" title="Link to this heading">#</a></h3> |
| <p>Map arrays can be constructed from lists of lists of tuples (key-item pairs), but only if |
| the type is explicitly passed into <a class="reference internal" href="generated/pyarrow.array.html#pyarrow.array" title="pyarrow.array"><code class="xref py py-meth docutils literal notranslate"><span class="pre">array()</span></code></a>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [56]: </span><span class="n">data</span> <span class="o">=</span> <span class="p">[[(</span><span class="s1">'x'</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="p">(</span><span class="s1">'y'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)],</span> <span class="p">[(</span><span class="s1">'a'</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="p">(</span><span class="s1">'b'</span><span class="p">,</span> <span class="mi">45</span><span class="p">)]]</span> |
| |
| <span class="gp">In [57]: </span><span class="n">ty</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">map_</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">(),</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">())</span> |
| |
| <span class="gp">In [58]: </span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">ty</span><span class="p">)</span> |
| <span class="gh">Out[58]: </span> |
| <span class="go"><pyarrow.lib.MapArray object at 0x7feff05a2a40></span> |
| <span class="go">[</span> |
| <span class="go"> keys:</span> |
| <span class="go"> [</span> |
| <span class="go"> "x",</span> |
| <span class="go"> "y"</span> |
| <span class="go"> ]</span> |
| <span class="go"> values:</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0</span> |
| <span class="go"> ],</span> |
| <span class="go"> keys:</span> |
| <span class="go"> [</span> |
| <span class="go"> "a",</span> |
| <span class="go"> "b"</span> |
| <span class="go"> ]</span> |
| <span class="go"> values:</span> |
| <span class="go"> [</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 45</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>MapArrays can also be constructed from offset, key, and item arrays. Offsets represent the |
| starting position of each map. Note that the <a class="reference internal" href="generated/pyarrow.MapArray.html#pyarrow.MapArray.keys" title="pyarrow.MapArray.keys"><code class="xref py py-attr docutils literal notranslate"><span class="pre">MapArray.keys</span></code></a> and <a class="reference internal" href="generated/pyarrow.MapArray.html#pyarrow.MapArray.items" title="pyarrow.MapArray.items"><code class="xref py py-attr docutils literal notranslate"><span class="pre">MapArray.items</span></code></a> |
| properties give the <em>flattened</em> keys and items. To keep the keys and items associated to |
| their row, use the <a class="reference internal" href="generated/pyarrow.ListArray.html#pyarrow.ListArray.from_arrays" title="pyarrow.ListArray.from_arrays"><code class="xref py py-meth docutils literal notranslate"><span class="pre">ListArray.from_arrays()</span></code></a> constructor with the |
| <a class="reference internal" href="generated/pyarrow.MapArray.html#pyarrow.MapArray.offsets" title="pyarrow.MapArray.offsets"><code class="xref py py-attr docutils literal notranslate"><span class="pre">MapArray.offsets</span></code></a> property.</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [59]: </span><span class="n">arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">MapArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="s1">'x'</span><span class="p">,</span> <span class="s1">'y'</span><span class="p">,</span> <span class="s1">'z'</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">])</span> |
| |
| <span class="gp">In [60]: </span><span class="n">arr</span><span class="o">.</span><span class="n">keys</span> |
| <span class="gh">Out[60]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7feff05a2b00></span> |
| <span class="go">[</span> |
| <span class="go"> "x",</span> |
| <span class="go"> "y",</span> |
| <span class="go"> "z"</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [61]: </span><span class="n">arr</span><span class="o">.</span><span class="n">items</span> |
| <span class="gh">Out[61]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7feff05a2bc0></span> |
| <span class="go">[</span> |
| <span class="go"> 4,</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [62]: </span><span class="n">pa</span><span class="o">.</span><span class="n">ListArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">offsets</span><span class="p">,</span> <span class="n">arr</span><span class="o">.</span><span class="n">keys</span><span class="p">)</span> |
| <span class="gh">Out[62]: </span> |
| <span class="go"><pyarrow.lib.ListArray object at 0x7feff05a2e60></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> "x",</span> |
| <span class="go"> "y"</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> "z"</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [63]: </span><span class="n">pa</span><span class="o">.</span><span class="n">ListArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">offsets</span><span class="p">,</span> <span class="n">arr</span><span class="o">.</span><span class="n">items</span><span class="p">)</span> |
| <span class="gh">Out[63]: </span> |
| <span class="go"><pyarrow.lib.ListArray object at 0x7feff05a2e00></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> 4,</span> |
| <span class="go"> 5</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 6</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="union-arrays"> |
| <h3>Union arrays<a class="headerlink" href="#union-arrays" title="Link to this heading">#</a></h3> |
| <p>The union type represents a nested array type where each value can be one |
| (and only one) of a set of possible types. There are two possible |
| storage types for union arrays: sparse and dense.</p> |
| <p>In a sparse union array, each of the child arrays has the same length |
| as the resulting union array. They are adjuncted with a <code class="docutils literal notranslate"><span class="pre">int8</span></code> “types” |
| array that tells, for each value, from which child array it must be |
| selected:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [64]: </span><span class="n">xs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">])</span> |
| |
| <span class="gp">In [65]: </span><span class="n">ys</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">False</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| |
| <span class="gp">In [66]: </span><span class="n">types</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">())</span> |
| |
| <span class="gp">In [67]: </span><span class="n">union_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">UnionArray</span><span class="o">.</span><span class="n">from_sparse</span><span class="p">(</span><span class="n">types</span><span class="p">,</span> <span class="p">[</span><span class="n">xs</span><span class="p">,</span> <span class="n">ys</span><span class="p">])</span> |
| |
| <span class="gp">In [68]: </span><span class="n">union_arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[68]: </span><span class="go">SparseUnionType(sparse_union<0: int64=0, 1: bool=1>)</span> |
| |
| <span class="gp">In [69]: </span><span class="n">union_arr</span> |
| <span class="gh">Out[69]: </span> |
| <span class="go"><pyarrow.lib.UnionArray object at 0x7feff05a3280></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- type_ids: [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 1</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 0 type: int64</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6,</span> |
| <span class="go"> 7</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> false,</span> |
| <span class="go"> false,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>In a dense union array, you also pass, in addition to the <code class="docutils literal notranslate"><span class="pre">int8</span></code> “types” |
| array, a <code class="docutils literal notranslate"><span class="pre">int32</span></code> “offsets” array that tells, for each value, at |
| each offset in the selected child array it can be found:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [70]: </span><span class="n">xs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">])</span> |
| |
| <span class="gp">In [71]: </span><span class="n">ys</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| |
| <span class="gp">In [72]: </span><span class="n">types</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">())</span> |
| |
| <span class="gp">In [73]: </span><span class="n">offsets</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">())</span> |
| |
| <span class="gp">In [74]: </span><span class="n">union_arr</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">UnionArray</span><span class="o">.</span><span class="n">from_dense</span><span class="p">(</span><span class="n">types</span><span class="p">,</span> <span class="n">offsets</span><span class="p">,</span> <span class="p">[</span><span class="n">xs</span><span class="p">,</span> <span class="n">ys</span><span class="p">])</span> |
| |
| <span class="gp">In [75]: </span><span class="n">union_arr</span><span class="o">.</span><span class="n">type</span> |
| <span class="gh">Out[75]: </span><span class="go">DenseUnionType(dense_union<0: int64=0, 1: bool=1>)</span> |
| |
| <span class="gp">In [76]: </span><span class="n">union_arr</span> |
| <span class="gh">Out[76]: </span> |
| <span class="go"><pyarrow.lib.UnionArray object at 0x7feff0647ac0></span> |
| <span class="go">-- is_valid: all not null</span> |
| <span class="go">-- type_ids: [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 0</span> |
| <span class="go"> ]</span> |
| <span class="go">-- value_offsets: [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 0 type: int64</span> |
| <span class="go"> [</span> |
| <span class="go"> 5,</span> |
| <span class="go"> 6,</span> |
| <span class="go"> 7</span> |
| <span class="go"> ]</span> |
| <span class="go">-- child 1 type: bool</span> |
| <span class="go"> [</span> |
| <span class="go"> false,</span> |
| <span class="go"> true</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="dictionary-arrays"> |
| <span id="data-dictionary"></span><h3>Dictionary Arrays<a class="headerlink" href="#dictionary-arrays" title="Link to this heading">#</a></h3> |
| <p>The <strong>Dictionary</strong> type in PyArrow is a special array type that is similar to a |
| factor in R or a <code class="docutils literal notranslate"><span class="pre">pandas.Categorical</span></code>. It enables one or more record batches |
| in a file or stream to transmit integer <em>indices</em> referencing a shared |
| <strong>dictionary</strong> containing the distinct values in the logical array. This is |
| particularly often used with strings to save memory and improve performance.</p> |
| <p>The way that dictionaries are handled in the Apache Arrow format and the way |
| they appear in C++ and Python is slightly different. We define a special |
| <a class="reference internal" href="generated/pyarrow.DictionaryArray.html#pyarrow.DictionaryArray" title="pyarrow.DictionaryArray"><code class="xref py py-class docutils literal notranslate"><span class="pre">DictionaryArray</span></code></a> type with a corresponding dictionary type. Let’s |
| consider an example:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [77]: </span><span class="n">indices</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span> |
| |
| <span class="gp">In [78]: </span><span class="n">dictionary</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">])</span> |
| |
| <span class="gp">In [79]: </span><span class="n">dict_array</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">DictionaryArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dictionary</span><span class="p">)</span> |
| |
| <span class="gp">In [80]: </span><span class="n">dict_array</span> |
| <span class="gh">Out[80]: </span> |
| <span class="go"><pyarrow.lib.DictionaryArray object at 0x7feff05d8f20></span> |
| |
| <span class="go">-- dictionary:</span> |
| <span class="go"> [</span> |
| <span class="go"> "foo",</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz"</span> |
| <span class="go"> ]</span> |
| <span class="go">-- indices:</span> |
| <span class="go"> [</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> null,</span> |
| <span class="go"> 2</span> |
| <span class="go"> ]</span> |
| </pre></div> |
| </div> |
| <p>Here we have:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [81]: </span><span class="nb">print</span><span class="p">(</span><span class="n">dict_array</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> |
| <span class="go">dictionary<values=string, indices=int64, ordered=0></span> |
| |
| <span class="gp">In [82]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">indices</span> |
| <span class="gh">Out[82]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7feff05f4340></span> |
| <span class="go">[</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 0,</span> |
| <span class="go"> null,</span> |
| <span class="go"> 2</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [83]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">dictionary</span> |
| <span class="gh">Out[83]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7feff05f41c0></span> |
| <span class="go">[</span> |
| <span class="go"> "foo",</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz"</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>When using <a class="reference internal" href="generated/pyarrow.DictionaryArray.html#pyarrow.DictionaryArray" title="pyarrow.DictionaryArray"><code class="xref py py-class docutils literal notranslate"><span class="pre">DictionaryArray</span></code></a> with pandas, the analogue is |
| <code class="docutils literal notranslate"><span class="pre">pandas.Categorical</span></code> (more on this later):</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [84]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span> |
| <span class="gh">Out[84]: </span> |
| <span class="go">0 foo</span> |
| <span class="go">1 bar</span> |
| <span class="go">2 foo</span> |
| <span class="go">3 bar</span> |
| <span class="go">4 baz</span> |
| <span class="go">5 foo</span> |
| <span class="go">6 NaN</span> |
| <span class="go">7 baz</span> |
| <span class="go">dtype: category</span> |
| <span class="go">Categories (3, object): ['foo', 'bar', 'baz']</span> |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| <section id="record-batches"> |
| <span id="data-record-batch"></span><h2>Record Batches<a class="headerlink" href="#record-batches" title="Link to this heading">#</a></h2> |
| <p>A <strong>Record Batch</strong> in Apache Arrow is a collection of equal-length array |
| instances. Let’s consider a collection of arrays:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [85]: </span><span class="n">data</span> <span class="o">=</span> <span class="p">[</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">,</span> <span class="kc">None</span><span class="p">]),</span> |
| <span class="gp"> ....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="kc">True</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">False</span><span class="p">,</span> <span class="kc">True</span><span class="p">])</span> |
| <span class="gp"> ....: </span><span class="p">]</span> |
| <span class="gp"> ....: </span> |
| </pre></div> |
| </div> |
| <p>A record batch can be created from this list of arrays using |
| <code class="docutils literal notranslate"><span class="pre">RecordBatch.from_arrays</span></code>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [86]: </span><span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">[</span><span class="s1">'f0'</span><span class="p">,</span> <span class="s1">'f1'</span><span class="p">,</span> <span class="s1">'f2'</span><span class="p">])</span> |
| |
| <span class="gp">In [87]: </span><span class="n">batch</span><span class="o">.</span><span class="n">num_columns</span> |
| <span class="gh">Out[87]: </span><span class="go">3</span> |
| |
| <span class="gp">In [88]: </span><span class="n">batch</span><span class="o">.</span><span class="n">num_rows</span> |
| <span class="gh">Out[88]: </span><span class="go">4</span> |
| |
| <span class="gp">In [89]: </span><span class="n">batch</span><span class="o">.</span><span class="n">schema</span> |
| <span class="gh">Out[89]: </span> |
| <span class="go">f0: int64</span> |
| <span class="go">f1: string</span> |
| <span class="go">f2: bool</span> |
| |
| <span class="gp">In [90]: </span><span class="n">batch</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> |
| <span class="gh">Out[90]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7feff05f4880></span> |
| <span class="go">[</span> |
| <span class="go"> "foo",</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz",</span> |
| <span class="go"> null</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>A record batch can be sliced without copying memory like an array:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [91]: </span><span class="n">batch2</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">slice</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span> |
| |
| <span class="gp">In [92]: </span><span class="n">batch2</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> |
| <span class="gh">Out[92]: </span> |
| <span class="go"><pyarrow.lib.StringArray object at 0x7feff05f5480></span> |
| <span class="go">[</span> |
| <span class="go"> "bar",</span> |
| <span class="go"> "baz",</span> |
| <span class="go"> null</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="tables"> |
| <span id="data-table"></span><h2>Tables<a class="headerlink" href="#tables" title="Link to this heading">#</a></h2> |
| <p>The PyArrow <a class="reference internal" href="generated/pyarrow.Table.html#pyarrow.Table" title="pyarrow.Table"><code class="xref py py-class docutils literal notranslate"><span class="pre">Table</span></code></a> type is not part of the Apache Arrow |
| specification, but is rather a tool to help with wrangling multiple record |
| batches and array pieces as a single logical dataset. As a relevant example, we |
| may receive multiple small record batches in a socket stream, then need to |
| concatenate them into contiguous memory for use in NumPy or pandas. The Table |
| object makes this efficient without requiring additional memory copying.</p> |
| <p>Considering the record batch we created above, we can create a Table containing |
| one or more copies of the batch using <code class="docutils literal notranslate"><span class="pre">Table.from_batches</span></code>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [93]: </span><span class="n">batches</span> <span class="o">=</span> <span class="p">[</span><span class="n">batch</span><span class="p">]</span> <span class="o">*</span> <span class="mi">5</span> |
| |
| <span class="gp">In [94]: </span><span class="n">table</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="o">.</span><span class="n">from_batches</span><span class="p">(</span><span class="n">batches</span><span class="p">)</span> |
| |
| <span class="gp">In [95]: </span><span class="n">table</span> |
| <span class="gh">Out[95]: </span> |
| <span class="go">pyarrow.Table</span> |
| <span class="go">f0: int64</span> |
| <span class="go">f1: string</span> |
| <span class="go">f2: bool</span> |
| <span class="gt">----</span> |
| <span class="ne">f0</span>: [[1,2,3,4],[1,2,3,4],...,[1,2,3,4],[1,2,3,4]] |
| <span class="ne">f1</span>: [["foo","bar","baz",null],["foo","bar","baz",null],...,["foo","bar","baz",null],["foo","bar","baz",null]] |
| <span class="ne">f2</span>: [[true,null,false,true],[true,null,false,true],...,[true,null,false,true],[true,null,false,true]] |
| |
| <span class="gp">In [96]: </span><span class="n">table</span><span class="o">.</span><span class="n">num_rows</span> |
| <span class="gh">Out[96]: </span><span class="go">20</span> |
| </pre></div> |
| </div> |
| <p>The table’s columns are instances of <a class="reference internal" href="generated/pyarrow.ChunkedArray.html#pyarrow.ChunkedArray" title="pyarrow.ChunkedArray"><code class="xref py py-class docutils literal notranslate"><span class="pre">ChunkedArray</span></code></a>, which is a |
| container for one or more arrays of the same type.</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [97]: </span><span class="n">c</span> <span class="o">=</span> <span class="n">table</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> |
| |
| <span class="gp">In [98]: </span><span class="n">c</span> |
| <span class="gh">Out[98]: </span> |
| <span class="go"><pyarrow.lib.ChunkedArray object at 0x7feff05f5a80></span> |
| <span class="go">[</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go">...,</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ],</span> |
| <span class="go"> [</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go"> ]</span> |
| <span class="go">]</span> |
| |
| <span class="gp">In [99]: </span><span class="n">c</span><span class="o">.</span><span class="n">num_chunks</span> |
| <span class="gh">Out[99]: </span><span class="go">5</span> |
| |
| <span class="gp">In [100]: </span><span class="n">c</span><span class="o">.</span><span class="n">chunk</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> |
| <span class="gh">Out[100]: </span> |
| <span class="go"><pyarrow.lib.Int64Array object at 0x7feff06462c0></span> |
| <span class="go">[</span> |
| <span class="go"> 1,</span> |
| <span class="go"> 2,</span> |
| <span class="go"> 3,</span> |
| <span class="go"> 4</span> |
| <span class="go">]</span> |
| </pre></div> |
| </div> |
| <p>As you’ll see in the <a class="reference internal" href="pandas.html#pandas-interop"><span class="std std-ref">pandas section</span></a>, we can convert |
| these objects to contiguous NumPy arrays for use in pandas:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [101]: </span><span class="n">c</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span> |
| <span class="gh">Out[101]: </span> |
| <span class="go">0 1</span> |
| <span class="go">1 2</span> |
| <span class="go">2 3</span> |
| <span class="go">3 4</span> |
| <span class="go">4 1</span> |
| <span class="go">5 2</span> |
| <span class="go">6 3</span> |
| <span class="go">7 4</span> |
| <span class="go">8 1</span> |
| <span class="go">9 2</span> |
| <span class="go">10 3</span> |
| <span class="go">11 4</span> |
| <span class="go">12 1</span> |
| <span class="go">13 2</span> |
| <span class="go">14 3</span> |
| <span class="go">15 4</span> |
| <span class="go">16 1</span> |
| <span class="go">17 2</span> |
| <span class="go">18 3</span> |
| <span class="go">19 4</span> |
| <span class="go">Name: f0, dtype: int64</span> |
| </pre></div> |
| </div> |
| <p>Multiple tables can also be concatenated together to form a single table using |
| <code class="docutils literal notranslate"><span class="pre">pyarrow.concat_tables</span></code>, if the schemas are equal:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [102]: </span><span class="n">tables</span> <span class="o">=</span> <span class="p">[</span><span class="n">table</span><span class="p">]</span> <span class="o">*</span> <span class="mi">2</span> |
| |
| <span class="gp">In [103]: </span><span class="n">table_all</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">concat_tables</span><span class="p">(</span><span class="n">tables</span><span class="p">)</span> |
| |
| <span class="gp">In [104]: </span><span class="n">table_all</span><span class="o">.</span><span class="n">num_rows</span> |
| <span class="gh">Out[104]: </span><span class="go">40</span> |
| |
| <span class="gp">In [105]: </span><span class="n">c</span> <span class="o">=</span> <span class="n">table_all</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> |
| |
| <span class="gp">In [106]: </span><span class="n">c</span><span class="o">.</span><span class="n">num_chunks</span> |
| <span class="gh">Out[106]: </span><span class="go">10</span> |
| </pre></div> |
| </div> |
| <p>This is similar to <code class="docutils literal notranslate"><span class="pre">Table.from_batches</span></code>, but uses tables as input instead of |
| record batches. Record batches can be made into tables, but not the other way |
| around, so if your data is already in table form, then use |
| <code class="docutils literal notranslate"><span class="pre">pyarrow.concat_tables</span></code>.</p> |
| </section> |
| <section id="custom-schema-and-field-metadata"> |
| <h2>Custom Schema and Field Metadata<a class="headerlink" href="#custom-schema-and-field-metadata" title="Link to this heading">#</a></h2> |
| <p>Arrow supports both schema-level and field-level custom key-value metadata |
| allowing for systems to insert their own application defined metadata to |
| customize behavior.</p> |
| <p>Custom metadata can be accessed at <a class="reference internal" href="generated/pyarrow.Schema.html#pyarrow.Schema.metadata" title="pyarrow.Schema.metadata"><code class="xref py py-attr docutils literal notranslate"><span class="pre">Schema.metadata</span></code></a> for the schema-level |
| and <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field.metadata" title="pyarrow.Field.metadata"><code class="xref py py-attr docutils literal notranslate"><span class="pre">Field.metadata</span></code></a> for the field-level.</p> |
| <p>Note that this metadata is preserved in <a class="reference internal" href="ipc.html#ipc"><span class="std std-ref">Streaming, Serialization, and IPC</span></a> processes.</p> |
| <p>To customize the schema metadata of an existing table you can use |
| <a class="reference internal" href="generated/pyarrow.Table.html#pyarrow.Table.replace_schema_metadata" title="pyarrow.Table.replace_schema_metadata"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Table.replace_schema_metadata()</span></code></a>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [107]: </span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">metadata</span> <span class="c1"># empty</span> |
| |
| <span class="gp">In [108]: </span><span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">replace_schema_metadata</span><span class="p">({</span><span class="s2">"f0"</span><span class="p">:</span> <span class="s2">"First dose"</span><span class="p">})</span> |
| |
| <span class="gp">In [109]: </span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[109]: </span><span class="go">{b'f0': b'First dose'}</span> |
| </pre></div> |
| </div> |
| <p>To customize the metadata of the field from the table schema you can use |
| <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field.with_metadata" title="pyarrow.Field.with_metadata"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Field.with_metadata()</span></code></a>:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [110]: </span><span class="n">field_f1</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"f1"</span><span class="p">)</span> |
| |
| <span class="gp">In [111]: </span><span class="n">field_f1</span><span class="o">.</span><span class="n">metadata</span> <span class="c1"># empty</span> |
| |
| <span class="gp">In [112]: </span><span class="n">field_f1</span> <span class="o">=</span> <span class="n">field_f1</span><span class="o">.</span><span class="n">with_metadata</span><span class="p">({</span><span class="s2">"f1"</span><span class="p">:</span> <span class="s2">"Second dose"</span><span class="p">})</span> |
| |
| <span class="gp">In [113]: </span><span class="n">field_f1</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[113]: </span><span class="go">{b'f1': b'Second dose'}</span> |
| </pre></div> |
| </div> |
| <p>Both options create a shallow copy of the data and do not in fact change the |
| Schema which is immutable. To change the metadata in the schema of the table |
| we created a new object when calling <a class="reference internal" href="generated/pyarrow.Table.html#pyarrow.Table.replace_schema_metadata" title="pyarrow.Table.replace_schema_metadata"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Table.replace_schema_metadata()</span></code></a>.</p> |
| <p>To change the metadata of the field in the schema we would need to define |
| a new schema and cast the data to this schema:</p> |
| <div class="highlight-ipython notranslate"><div class="highlight"><pre><span></span><span class="gp">In [114]: </span><span class="n">my_schema2</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([</span> |
| <span class="gp"> .....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'f0'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">(),</span> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"First dose"</span><span class="p">}),</span> |
| <span class="gp"> .....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'f1'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">(),</span> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"name"</span><span class="p">:</span> <span class="s2">"Second dose"</span><span class="p">}),</span> |
| <span class="gp"> .....: </span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'f2'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">())],</span> |
| <span class="gp"> .....: </span> <span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"f2"</span><span class="p">:</span> <span class="s2">"booster"</span><span class="p">})</span> |
| <span class="gp"> .....: </span> |
| |
| <span class="gp">In [115]: </span><span class="n">t2</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">my_schema2</span><span class="p">)</span> |
| |
| <span class="gp">In [116]: </span><span class="n">t2</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"f0"</span><span class="p">)</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[116]: </span><span class="go">{b'name': b'First dose'}</span> |
| |
| <span class="gp">In [117]: </span><span class="n">t2</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s2">"f1"</span><span class="p">)</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[117]: </span><span class="go">{b'name': b'Second dose'}</span> |
| |
| <span class="gp">In [118]: </span><span class="n">t2</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">metadata</span> |
| <span class="gh">Out[118]: </span><span class="go">{b'f2': b'booster'}</span> |
| </pre></div> |
| </div> |
| <p>Metadata key and value pairs are <code class="docutils literal notranslate"><span class="pre">std::string</span></code> objects in the C++ implementation |
| and so they are bytes objects (<code class="docutils literal notranslate"><span class="pre">b'...'</span></code>) in Python.</p> |
| </section> |
| <section id="record-batch-readers"> |
| <h2>Record Batch Readers<a class="headerlink" href="#record-batch-readers" title="Link to this heading">#</a></h2> |
| <p>Many functions in PyArrow either return or take as an argument a <a class="reference internal" href="generated/pyarrow.RecordBatchReader.html#pyarrow.RecordBatchReader" title="pyarrow.RecordBatchReader"><code class="xref py py-class docutils literal notranslate"><span class="pre">RecordBatchReader</span></code></a>. |
| It can be used like any iterable of record batches, but also provides their common |
| schema without having to get any of the batches.:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">schema</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([(</span><span class="s1">'x'</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">())])</span> |
| <span class="gp">>>> </span><span class="k">def</span><span class="w"> </span><span class="nf">iter_record_batches</span><span class="p">():</span> |
| <span class="gp">... </span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">2</span><span class="p">):</span> |
| <span class="gp">... </span> <span class="k">yield</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">([</span><span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])],</span> <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">reader</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchReader</span><span class="o">.</span><span class="n">from_batches</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">iter_record_batches</span><span class="p">())</span> |
| <span class="gp">>>> </span><span class="nb">print</span><span class="p">(</span><span class="n">reader</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> |
| <span class="go">pyarrow.Schema</span> |
| <span class="go">x: int64</span> |
| <span class="gp">>>> </span><span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">reader</span><span class="p">:</span> |
| <span class="gp">... </span> <span class="nb">print</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span> |
| <span class="go">pyarrow.RecordBatch</span> |
| <span class="go">x: int64</span> |
| <span class="go">pyarrow.RecordBatch</span> |
| <span class="go">x: int64</span> |
| </pre></div> |
| </div> |
| <p>It can also be sent between languages using the <a class="reference internal" href="../format/CStreamInterface.html#c-stream-interface"><span class="std std-ref">C stream interface</span></a>.</p> |
| </section> |
| <section id="conversion-of-recordbatch-to-tensor"> |
| <h2>Conversion of RecordBatch to Tensor<a class="headerlink" href="#conversion-of-recordbatch-to-tensor" title="Link to this heading">#</a></h2> |
| <p>Each array of the <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> has it’s own contiguous memory that is not necessarily |
| adjacent to other arrays. A different memory structure that is used in machine learning |
| libraries is a two dimensional array (also called a 2-dim tensor or a matrix) which takes |
| only one contiguous block of memory.</p> |
| <p>For this reason there is a function <code class="docutils literal notranslate"><span class="pre">pyarrow.RecordBatch.to_tensor()</span></code> available |
| to efficiently convert tabular columnar data into a tensor.</p> |
| <p>Data types supported in this conversion are unsigned, signed integer and float |
| types. Currently only column-major conversion is supported.</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span> <span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span> <span class="n">arr1</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">]</span> |
| <span class="gp">>>> </span> <span class="n">arr2</span> <span class="o">=</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">,</span> <span class="mi">50</span><span class="p">]</span> |
| <span class="gp">>>> </span> <span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">arr1</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">uint16</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">arr2</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int16</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="p">],</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">]</span> |
| <span class="gp">... </span> <span class="p">)</span> |
| <span class="gp">>>> </span> <span class="n">batch</span><span class="o">.</span><span class="n">to_tensor</span><span class="p">()</span> |
| <span class="go"><pyarrow.Tensor></span> |
| <span class="go">type: int32</span> |
| <span class="go">shape: (9, 2)</span> |
| <span class="go">strides: (4, 36)</span> |
| <span class="gp">>>> </span> <span class="n">batch</span><span class="o">.</span><span class="n">to_tensor</span><span class="p">()</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span> |
| <span class="go">array([[ 1, 10],</span> |
| <span class="go"> [ 2, 20],</span> |
| <span class="go"> [ 3, 30],</span> |
| <span class="go"> [ 4, 40],</span> |
| <span class="go"> [ 5, 50]], dtype=int32)</span> |
| </pre></div> |
| </div> |
| <p>With <code class="docutils literal notranslate"><span class="pre">null_to_nan</span></code> set to <code class="docutils literal notranslate"><span class="pre">True</span></code> one can also convert data with |
| nulls. They will be converted to <code class="docutils literal notranslate"><span class="pre">NaN</span></code>:</p> |
| <div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span> |
| <span class="gp">>>> </span><span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">(</span> |
| <span class="gp">... </span> <span class="p">[</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">float32</span><span class="p">()),</span> |
| <span class="gp">... </span> <span class="p">],</span> <span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span><span class="p">]</span> |
| <span class="gp">... </span><span class="p">)</span> |
| <span class="gp">>>> </span><span class="n">batch</span><span class="o">.</span><span class="n">to_tensor</span><span class="p">(</span><span class="n">null_to_nan</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">()</span> |
| <span class="go">array([[ 1., 10.],</span> |
| <span class="go"> [ 2., 20.],</span> |
| <span class="go"> [ 3., 30.],</span> |
| <span class="go"> [ 4., 40.],</span> |
| <span class="go"> [nan, nan]])</span> |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="getstarted.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Getting Started</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="compute.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Compute Functions</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <dialog id="pst-secondary-sidebar-modal"></dialog> |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#type-metadata">Type Metadata</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#schemas">Schemas</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#arrays">Arrays</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#none-values-and-nan-handling">None values and NAN handling</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#list-arrays">List arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#listview-arrays">ListView arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#struct-arrays">Struct arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#map-arrays">Map arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#union-arrays">Union arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#dictionary-arrays">Dictionary Arrays</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#record-batches">Record Batches</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#tables">Tables</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#custom-schema-and-field-metadata">Custom Schema and Field Metadata</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#record-batch-readers">Record Batch Readers</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#conversion-of-recordbatch-to-tensor">Conversion of RecordBatch to Tensor</a></li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/python/data.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script> |
| <script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2025 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| <!-- # L10n: Setting the PST URL as an argument as this does not need to be localized --> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.16.1. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
