| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../../" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <title>Developing PyArrow — Apache Arrow v23.0.0.dev37</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || ""; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| <link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=03e43079" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/copybutton.css?v=76b2166b" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/sphinx-design.min.css?v=95c83b7e" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=8dcd28dc" /> |
| |
| <!-- So that users can add custom icons --> |
| <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" /> |
| <link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" /> |
| |
| <script src="../../_static/documentation_options.js?v=9fc6757a"></script> |
| <script src="../../_static/doctools.js?v=9bcbadda"></script> |
| <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script> |
| <script src="../../_static/clipboard.min.js?v=a7894cd8"></script> |
| <script src="../../_static/copybutton.js?v=3bb21c8c"></script> |
| <script src="../../_static/design-tabs.js?v=f930bc37"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'developers/python/development';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.16.1'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = |
| true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/developers/python/development.html" /> |
| <link rel="icon" href="../../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../../genindex.html" /> |
| <link rel="search" title="Search" href="../../search.html" /> |
| <link rel="next" title="Continuous Integration" href="../continuous_integration/index.html" /> |
| <link rel="prev" title="Building PyArrow" href="building.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="23.0.0.dev37" /> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class=" navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v23.0.0.dev37 - Home"/> |
| <img src="../../_static/arrow-dark.png" class="logo__image only-dark pst-js-only" alt="Apache Arrow v23.0.0.dev37 - Home"/> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class=" navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> |
| <span class="fa-solid fa-outdent"></span> |
| </button> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="../bug_reports.html">Bug reports and feature requests</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../guide/index.html">New Contributor’s Guide</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/architectural_overview.html">Architectural Overview</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/communication.html">Communication</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../guide/step_by_step/index.html">Steps in making your first PR</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/set_up.html">Set up</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/building.html">Building the Arrow libraries 🏋🏿♀️</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/finding_issues.html">Finding good first issues 🔎</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/arrow_codebase.html">Working on the Arrow codebase 🧐</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/testing.html">Testing 🧪</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/styling.html">Styling 😎</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/pr_lifecycle.html">Lifecycle of a pull request</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/documentation.html">Helping with documentation</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../guide/tutorials/index.html">Tutorials</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/tutorials/python_tutorial.html">Python tutorial</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/tutorials/r_tutorial.html">R tutorials</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/resources.html">Additional information and resources</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1"><a class="reference internal" href="../overview.html">Contributing Overview</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../reviewing.html">Reviewing contributions</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../cpp/index.html">C++ Development</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/building.html">Building Arrow C++</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/development.html">Development Guidelines</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/windows.html">Developing on Windows</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/emscripten.html">Cross compiling for WebAssembly with Emscripten</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/conventions.html">Conventions</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/fuzzing.html">Fuzzing Arrow C++</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/compute.html">Developing Arrow C++ Compute</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../cpp/acero.html">Developing Acero</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/acero/swiss_table.html">Swiss Table</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../java/index.html">Java Development</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../java/building.html">Building Arrow Java</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/development.html">Development Guidelines</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 current active has-children"><a class="reference internal" href="index.html">Python Development</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l2"><a class="reference internal" href="building.html">Building PyArrow</a></li> |
| <li class="toctree-l2 current active"><a class="current reference internal" href="#">Developing PyArrow</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../continuous_integration/index.html">Continuous Integration</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../continuous_integration/overview.html">Continuous Integration</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../continuous_integration/docker.html">Running Docker Builds</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../continuous_integration/archery.html">Daily Development using Archery</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../continuous_integration/crossbow.html">Packaging and Testing with Crossbow</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1"><a class="reference internal" href="../benchmarks.html">Benchmarks</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../documentation.html">Building the Documentation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../release.html">Release Management Guide</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../release_verification.html">Release Verification Process</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Development</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Python Development</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Developing PyArrow</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="developing-pyarrow"> |
| <span id="develop-pyarrow"></span><h1>Developing PyArrow<a class="headerlink" href="#developing-pyarrow" title="Link to this heading">#</a></h1> |
| <section id="coding-style"> |
| <span id="python-coding-style"></span><h2>Coding Style<a class="headerlink" href="#coding-style" title="Link to this heading">#</a></h2> |
| <p>We follow a similar PEP8-like coding style to the <a class="reference external" href="https://github.com/pandas-dev/pandas">pandas project</a>. To fix style issues, use the |
| <code class="docutils literal notranslate"><span class="pre">pre-commit</span></code> command:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>pre-commit<span class="w"> </span>run<span class="w"> </span>--show-diff-on-failure<span class="w"> </span>--color<span class="o">=</span>always<span class="w"> </span>--all-files<span class="w"> </span>python |
| </pre></div> |
| </div> |
| </section> |
| <section id="unit-testing"> |
| <span id="python-unit-testing"></span><h2>Unit Testing<a class="headerlink" href="#unit-testing" title="Link to this heading">#</a></h2> |
| <p>We are using <a class="reference external" href="https://docs.pytest.org/en/latest/">pytest</a> to develop our unit |
| test suite. After <a class="reference external" href="build_pyarrow">building the project</a> you can run its unit tests |
| like so:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nb">pushd</span><span class="w"> </span>arrow/python |
| <span class="gp">$ </span>python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>pyarrow |
| <span class="gp">$ </span><span class="nb">popd</span> |
| </pre></div> |
| </div> |
| <p>Package requirements to run the unit tests are found in |
| <code class="docutils literal notranslate"><span class="pre">requirements-test.txt</span></code> and can be installed if needed with <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">-r</span> |
| <span class="pre">requirements-test.txt</span></code>.</p> |
| <p>If you get import errors for <code class="docutils literal notranslate"><span class="pre">pyarrow._lib</span></code> or another PyArrow module when |
| trying to run the tests, run <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-m</span> <span class="pre">pytest</span> <span class="pre">arrow/python/pyarrow</span></code> and check |
| if the editable version of pyarrow was installed correctly.</p> |
| <p>The project has a number of custom command line options for its test |
| suite. Some tests are disabled by default, for example. To see all the options, |
| run</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>pyarrow<span class="w"> </span>--help |
| </pre></div> |
| </div> |
| <p>and look for the “custom options” section.</p> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>There are a few low-level tests written directly in C++. These tests are |
| implemented in <a class="reference external" href="https://github.com/apache/arrow/blob/main/python/pyarrow/src/arrow/python/python_test.cc">pyarrow/src/arrow/python/python_test.cc</a>, |
| but they are also wrapped in a <code class="docutils literal notranslate"><span class="pre">pytest</span></code>-based |
| <a class="reference external" href="https://github.com/apache/arrow/blob/main/python/pyarrow/tests/test_cpp_internals.py">test module</a> |
| run automatically as part of the PyArrow test suite.</p> |
| </div> |
| <section id="test-groups"> |
| <h3>Test Groups<a class="headerlink" href="#test-groups" title="Link to this heading">#</a></h3> |
| <p>We have many tests that are grouped together using pytest marks. Some of these |
| are disabled by default. To enable a test group, pass <code class="docutils literal notranslate"><span class="pre">--$GROUP_NAME</span></code>, |
| e.g. <code class="docutils literal notranslate"><span class="pre">--parquet</span></code>. To disable a test group, prepend <code class="docutils literal notranslate"><span class="pre">disable</span></code>, so |
| <code class="docutils literal notranslate"><span class="pre">--disable-parquet</span></code> for example. To run <strong>only</strong> the unit tests for a |
| particular group, prepend <code class="docutils literal notranslate"><span class="pre">only-</span></code> instead, for example <code class="docutils literal notranslate"><span class="pre">--only-parquet</span></code>.</p> |
| <p>The test groups currently include:</p> |
| <ul class="simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">dataset</span></code>: Apache Arrow Dataset tests</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">flight</span></code>: Flight RPC tests</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">gandiva</span></code>: tests for Gandiva expression compiler (uses LLVM)</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">hdfs</span></code>: tests that use libhdfs to access the Hadoop filesystem</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">hypothesis</span></code>: tests that use the <code class="docutils literal notranslate"><span class="pre">hypothesis</span></code> module for generating |
| random test cases. Note that <code class="docutils literal notranslate"><span class="pre">--hypothesis</span></code> doesn’t work due to a quirk |
| with pytest, so you have to pass <code class="docutils literal notranslate"><span class="pre">--enable-hypothesis</span></code></p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">large_memory</span></code>: Test requiring a large amount of system RAM</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">orc</span></code>: Apache ORC tests</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">parquet</span></code>: Apache Parquet tests</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">s3</span></code>: Tests for Amazon S3</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">tensorflow</span></code>: Tests that involve TensorFlow</p></li> |
| </ul> |
| </section> |
| </section> |
| <section id="doctest"> |
| <h2>Doctest<a class="headerlink" href="#doctest" title="Link to this heading">#</a></h2> |
| <p>We are using <a class="reference external" href="https://docs.python.org/3/library/doctest.html">doctest</a> |
| to check that docstring examples are up-to-date and correct. You can |
| also do that locally by running:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nb">pushd</span><span class="w"> </span>arrow/python |
| <span class="gp">$ </span>python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>--doctest-modules |
| <span class="gp">$ </span>python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>--doctest-modules<span class="w"> </span>path/to/module.py<span class="w"> </span><span class="c1"># checking single file</span> |
| <span class="gp">$ </span><span class="nb">popd</span> |
| </pre></div> |
| </div> |
| <p>for <code class="docutils literal notranslate"><span class="pre">.py</span></code> files or</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span><span class="nb">pushd</span><span class="w"> </span>arrow/python |
| <span class="gp">$ </span>python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>--doctest-cython |
| <span class="gp">$ </span>python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>--doctest-cython<span class="w"> </span>path/to/module.pyx<span class="w"> </span><span class="c1"># checking single file</span> |
| <span class="gp">$ </span><span class="nb">popd</span> |
| </pre></div> |
| </div> |
| <p>for <code class="docutils literal notranslate"><span class="pre">.pyx</span></code> and <code class="docutils literal notranslate"><span class="pre">.pxi</span></code> files. In this case you will also need to |
| install the <a class="reference external" href="https://github.com/lgpage/pytest-cython">pytest-cython</a> plugin.</p> |
| </section> |
| <section id="debugging"> |
| <h2>Debugging<a class="headerlink" href="#debugging" title="Link to this heading">#</a></h2> |
| <section id="debug-build"> |
| <h3>Debug build<a class="headerlink" href="#debug-build" title="Link to this heading">#</a></h3> |
| <p>Since PyArrow depends on the Arrow C++ libraries, debugging can |
| frequently involve crossing between Python and C++ shared libraries. |
| For the best experience, make sure you’ve built both Arrow C++ |
| (<code class="docutils literal notranslate"><span class="pre">-DCMAKE_BUILD_TYPE=Debug</span></code>) and PyArrow (<code class="docutils literal notranslate"><span class="pre">export</span> <span class="pre">PYARROW_BUILD_TYPE=debug</span></code>) |
| in debug mode.</p> |
| </section> |
| <section id="using-gdb-on-linux"> |
| <h3>Using gdb on Linux<a class="headerlink" href="#using-gdb-on-linux" title="Link to this heading">#</a></h3> |
| <p>To debug the C++ libraries with gdb while running the Python unit |
| tests, first start pytest with gdb:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>gdb<span class="w"> </span>--args<span class="w"> </span>python<span class="w"> </span>-m<span class="w"> </span>pytest<span class="w"> </span>pyarrow/tests/test_to_run.py<span class="w"> </span>-k<span class="w"> </span><span class="nv">$TEST_TO_MATCH</span> |
| </pre></div> |
| </div> |
| <p>To set a breakpoint, use the same gdb syntax that you would when |
| debugging a C++ program, for example:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp gp-VirtualEnv">(gdb)</span> <span class="go">b src/arrow/python/arrow_to_pandas.cc:1874</span> |
| <span class="go">No source file named src/arrow/python/arrow_to_pandas.cc.</span> |
| <span class="go">Make breakpoint pending on future shared library load? (y or [n]) y</span> |
| <span class="go">Breakpoint 1 (src/arrow/python/arrow_to_pandas.cc:1874) pending.</span> |
| </pre></div> |
| </div> |
| <div class="admonition seealso"> |
| <p class="admonition-title">See also</p> |
| <p>The <a class="reference internal" href="../../cpp/gdb.html#cpp-gdb-extension"><span class="std std-ref">GDB extension for Arrow C++</span></a>.</p> |
| </div> |
| <p>Similarly, use lldb when debugging on macOS.</p> |
| </section> |
| </section> |
| <section id="benchmarking"> |
| <h2>Benchmarking<a class="headerlink" href="#benchmarking" title="Link to this heading">#</a></h2> |
| <p>For running the benchmarks, see <a class="reference internal" href="../../python/benchmarks.html#python-benchmarks"><span class="std std-ref">Benchmarks</span></a>.</p> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="building.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Building PyArrow</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="../continuous_integration/index.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Continuous Integration</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <dialog id="pst-secondary-sidebar-modal"></dialog> |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#coding-style">Coding Style</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#unit-testing">Unit Testing</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#test-groups">Test Groups</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#doctest">Doctest</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#debugging">Debugging</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#debug-build">Debug build</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#using-gdb-on-linux">Using gdb on Linux</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#benchmarking">Benchmarking</a></li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/developers/python/development.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script> |
| <script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2025 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| <!-- # L10n: Setting the PST URL as an argument as this does not need to be localized --> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.16.1. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
