| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../../" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <title>Packaging and Testing with Crossbow — Apache Arrow v23.0.0.dev37</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || ""; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| <link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=03e43079" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/copybutton.css?v=76b2166b" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/sphinx-design.min.css?v=95c83b7e" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=8dcd28dc" /> |
| |
| <!-- So that users can add custom icons --> |
| <script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" /> |
| <link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" /> |
| |
| <script src="../../_static/documentation_options.js?v=9fc6757a"></script> |
| <script src="../../_static/doctools.js?v=9bcbadda"></script> |
| <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script> |
| <script src="../../_static/clipboard.min.js?v=a7894cd8"></script> |
| <script src="../../_static/copybutton.js?v=3bb21c8c"></script> |
| <script src="../../_static/design-tabs.js?v=f930bc37"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'developers/continuous_integration/crossbow';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.16.1'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = |
| true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/developers/continuous_integration/crossbow.html" /> |
| <link rel="icon" href="../../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../../genindex.html" /> |
| <link rel="search" title="Search" href="../../search.html" /> |
| <link rel="next" title="Benchmarks" href="../benchmarks.html" /> |
| <link rel="prev" title="Daily Development using Archery" href="archery.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="23.0.0.dev37" /> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class=" navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v23.0.0.dev37 - Home"/> |
| <img src="../../_static/arrow-dark.png" class="logo__image only-dark pst-js-only" alt="Apache Arrow v23.0.0.dev37 - Home"/> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class=" navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> |
| <span class="fa-solid fa-outdent"></span> |
| </button> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../index.html"> |
| Development |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../../implementations.html"> |
| Implementations |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"><div class="kapa-ai-bot"> |
| <script |
| async |
| src="https://widget.kapa.ai/kapa-widget.bundle.js" |
| data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2" |
| data-project-name="Apache Arrow" |
| data-project-color="#000000" |
| data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png" |
| data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc." |
| data-consent-required="true" |
| data-user-analytics-cookie-enabled="false" |
| data-consent-screen-disclaimer="By clicking "I agree, let's chat", you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies." |
| ></script> |
| |
| </div> |
| |
| </div> |
| |
| <div class="navbar-item"> |
| <div class="version-switcher__container dropdown pst-js-only"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div></div> |
| |
| <div class="navbar-item"> |
| |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i> |
| </button></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">LinkedIn</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i> |
| <span class="sr-only">BlueSky</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="../bug_reports.html">Bug reports and feature requests</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../guide/index.html">New Contributor’s Guide</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/architectural_overview.html">Architectural Overview</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/communication.html">Communication</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../guide/step_by_step/index.html">Steps in making your first PR</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/set_up.html">Set up</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/building.html">Building the Arrow libraries 🏋🏿♀️</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/finding_issues.html">Finding good first issues 🔎</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/arrow_codebase.html">Working on the Arrow codebase 🧐</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/testing.html">Testing 🧪</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/styling.html">Styling 😎</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/step_by_step/pr_lifecycle.html">Lifecycle of a pull request</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/documentation.html">Helping with documentation</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../guide/tutorials/index.html">Tutorials</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/tutorials/python_tutorial.html">Python tutorial</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../guide/tutorials/r_tutorial.html">R tutorials</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l2"><a class="reference internal" href="../guide/resources.html">Additional information and resources</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1"><a class="reference internal" href="../overview.html">Contributing Overview</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../reviewing.html">Reviewing contributions</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../cpp/index.html">C++ Development</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/building.html">Building Arrow C++</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/development.html">Development Guidelines</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/windows.html">Developing on Windows</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/emscripten.html">Cross compiling for WebAssembly with Emscripten</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/conventions.html">Conventions</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/fuzzing.html">Fuzzing Arrow C++</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../cpp/compute.html">Developing Arrow C++ Compute</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../cpp/acero.html">Developing Acero</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../cpp/acero/swiss_table.html">Swiss Table</a></li> |
| </ul> |
| </details></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../java/index.html">Java Development</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../java/building.html">Building Arrow Java</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../java/development.html">Development Guidelines</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../python/index.html">Python Development</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../python/building.html">Building PyArrow</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../python/development.html">Developing PyArrow</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1 current active has-children"><a class="reference internal" href="index.html">Continuous Integration</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current"> |
| <li class="toctree-l2"><a class="reference internal" href="overview.html">Continuous Integration</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="docker.html">Running Docker Builds</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="archery.html">Daily Development using Archery</a></li> |
| <li class="toctree-l2 current active"><a class="current reference internal" href="#">Packaging and Testing with Crossbow</a></li> |
| </ul> |
| </details></li> |
| <li class="toctree-l1"><a class="reference internal" href="../benchmarks.html">Benchmarks</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../documentation.html">Building the Documentation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../release.html">Release Management Guide</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../release_verification.html">Release Verification Process</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Development</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Continuous Integration</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Packaging and Testing with Crossbow</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="packaging-and-testing-with-crossbow"> |
| <span id="crossbow"></span><h1>Packaging and Testing with Crossbow<a class="headerlink" href="#packaging-and-testing-with-crossbow" title="Link to this heading">#</a></h1> |
| <p>The content of <code class="docutils literal notranslate"><span class="pre">arrow/dev/tasks</span></code> directory aims for automating the process of |
| Arrow packaging and integration testing.</p> |
| <dl class="simple"> |
| <dt>Packages:</dt><dd><ul class="simple"> |
| <li><p>C++ and Python <a class="reference external" href="conda-recipes">conda-forge packages</a> for Linux, macOS and Windows</p></li> |
| <li><p>Python <a class="reference external" href="python-wheels">Wheels</a> for Linux, macOS and Windows</p></li> |
| <li><p>C++ and GLib <a class="reference external" href="linux-packages">Linux packages</a> for multiple distributions</p></li> |
| <li><p>Java for Gandiva</p></li> |
| </ul> |
| </dd> |
| <dt>Integration tests:</dt><dd><ul class="simple"> |
| <li><p>Various docker tests</p></li> |
| <li><p>Pandas</p></li> |
| <li><p>Dask</p></li> |
| <li><p>Turbodbc</p></li> |
| <li><p>HDFS</p></li> |
| <li><p>Spark</p></li> |
| </ul> |
| </dd> |
| </dl> |
| <section id="architecture"> |
| <h2>Architecture<a class="headerlink" href="#architecture" title="Link to this heading">#</a></h2> |
| <section id="executors"> |
| <h3>Executors<a class="headerlink" href="#executors" title="Link to this heading">#</a></h3> |
| <p>Individual jobs are executed on public CI services, currently:</p> |
| <ul class="simple"> |
| <li><p>Linux: GitHub Actions, Travis CI, Azure Pipelines</p></li> |
| <li><p>macOS: GitHub Actions, Azure Pipelines</p></li> |
| <li><p>Windows: GitHub Actions, Azure Pipelines</p></li> |
| </ul> |
| </section> |
| <section id="queue"> |
| <h3>Queue<a class="headerlink" href="#queue" title="Link to this heading">#</a></h3> |
| <p>Because of the nature of how the CI services work, the scheduling of |
| jobs happens through an additional git repository, which acts like a job |
| queue for the tasks. Anyone can host a <code class="docutils literal notranslate"><span class="pre">queue</span></code> repository (usually |
| named <code class="docutils literal notranslate"><span class="pre"><ghuser>/crossbow</span></code>).</p> |
| <p>A job is a git commit on a particular git branch, containing the required |
| configuration files to run the requested builds (like <code class="docutils literal notranslate"><span class="pre">.travis.yml</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">azure-pipelines.yml</span></code>, or <code class="docutils literal notranslate"><span class="pre">crossbow.yml</span></code> for <a class="reference external" href="https://docs.github.com/en/actions/quickstart">GitHub Actions</a> ).</p> |
| </section> |
| <section id="scheduler"> |
| <h3>Scheduler<a class="headerlink" href="#scheduler" title="Link to this heading">#</a></h3> |
| <p>Crossbow handles version generation, task rendering and |
| submission. The tasks are defined in <code class="docutils literal notranslate"><span class="pre">tasks.yml</span></code>.</p> |
| </section> |
| </section> |
| <section id="install"> |
| <h2>Install<a class="headerlink" href="#install" title="Link to this heading">#</a></h2> |
| <p>The following guide depends on GitHub, but theoretically any git |
| server can be used.</p> |
| <p>If you are not using the <a class="reference external" href="https://github.com/ursacomputing/crossbow">ursacomputing/crossbow</a> |
| repository, you will need to complete the first two steps, otherwise proceed |
| to step 3:</p> |
| <ol class="arabic"> |
| <li><p><a class="reference external" href="https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-new-repository">Create the queue repository</a></p></li> |
| <li><p>Enable <a class="reference external" href="https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started/pipelines-sign-up">Azure Pipelines</a> integrations for the newly created queue |
| repository.</p></li> |
| <li><p>Clone either <a class="reference external" href="https://github.com/ursacomputing/crossbow">ursacomputing/crossbow</a> if you are using that, or the newly |
| created repository next to the arrow repository:</p> |
| <p>By default the scripts looks for a <code class="docutils literal notranslate"><span class="pre">crossbow</span></code> clone next to the <code class="docutils literal notranslate"><span class="pre">arrow</span></code> |
| directory, but this can configured through command line arguments.</p> |
| <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>git<span class="w"> </span>clone<span class="w"> </span>https://github.com/<user>/crossbow<span class="w"> </span>crossbow |
| </pre></div> |
| </div> |
| <p><strong>Important note:</strong> Crossbow only supports GitHub token based |
| authentication. Although it overwrites the repository urls provided with ssh |
| protocol, it’s advisable to use the HTTPS repository URLs.</p> |
| </li> |
| <li><p><a class="reference external" href="https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/">Create a Personal Access Token</a> with <code class="docutils literal notranslate"><span class="pre">repo</span></code> and <code class="docutils literal notranslate"><span class="pre">workflow</span></code> permissions (other |
| permissions are not needed)</p></li> |
| <li><p>Locally export the token as an environment variable:</p> |
| <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span><span class="w"> </span><span class="nv">GH_TOKEN</span><span class="o">=</span><token> |
| </pre></div> |
| </div> |
| <p>or pass as an argument to the CLI script <code class="docutils literal notranslate"><span class="pre">--github-token</span></code></p> |
| </li> |
| <li><p>Install Python (minimum supported version is 3.10):</p> |
| <div class="line-block"> |
| <div class="line">Miniconda is preferred, see installation instructions:</div> |
| <div class="line"><a class="reference external" href="https://conda.io/docs/user-guide/install/index.html">https://conda.io/docs/user-guide/install/index.html</a></div> |
| </div> |
| </li> |
| <li><p>Install the archery toolset containing crossbow itself:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>pip<span class="w"> </span>install<span class="w"> </span>-e<span class="w"> </span><span class="s2">"arrow/dev/archery[crossbow]"</span> |
| </pre></div> |
| </div> |
| </li> |
| <li><p>Try running it:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>--help |
| </pre></div> |
| </div> |
| </li> |
| </ol> |
| </section> |
| <section id="usage"> |
| <h2>Usage<a class="headerlink" href="#usage" title="Link to this heading">#</a></h2> |
| <p>The script does the following:</p> |
| <ol class="arabic"> |
| <li><p>Detects the current repository, thus supports forks. The following |
| snippet will build kszucs’s fork instead of the upstream apache/arrow |
| repository.</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>git<span class="w"> </span>clone<span class="w"> </span>https://github.com/kszucs/arrow |
| <span class="gp">$ </span>git<span class="w"> </span>clone<span class="w"> </span>https://github.com/kszucs/crossbow |
| |
| <span class="gp">$ </span><span class="nb">cd</span><span class="w"> </span>arrow/dev/tasks |
| <span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>--help<span class="w"> </span><span class="c1"># show the available options</span> |
| <span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>conda-win<span class="w"> </span>conda-linux<span class="w"> </span>conda-osx |
| </pre></div> |
| </div> |
| </li> |
| <li><p>Gets the HEAD commit of the currently checked out branch and |
| generates the version number based on <a class="reference external" href="https://pypi.python.org/pypi/setuptools_scm">setuptools_scm</a>. So to build |
| a particular branch check out before running the script:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>git<span class="w"> </span>checkout<span class="w"> </span>ARROW-<ticket<span class="w"> </span>number> |
| <span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>--dry-run<span class="w"> </span>conda-linux<span class="w"> </span>conda-osx |
| </pre></div> |
| </div> |
| <p>Note that the arrow branch must be pushed beforehand, because the |
| script will clone the selected branch.</p> |
| </li> |
| <li><p>Reads and renders the required build configurations with the |
| parameters substituted.</p></li> |
| <li><p>Create a branch per task, prefixed with the job id. For example, to |
| build conda recipes on linux, it will create a new branch: |
| <code class="docutils literal notranslate"><span class="pre">crossbow@build-<id>-conda-linux</span></code>.</p></li> |
| <li><p>Pushes the modified branches to GitHub which triggers the builds. For |
| authentication it uses GitHub OAuth tokens described in the install |
| section.</p></li> |
| </ol> |
| <section id="query-the-build-status"> |
| <h3>Query the build status<a class="headerlink" href="#query-the-build-status" title="Link to this heading">#</a></h3> |
| <p>Build id (which has a corresponding branch in the queue repository) is returned |
| by the <code class="docutils literal notranslate"><span class="pre">submit</span></code> command.</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>status<span class="w"> </span><build<span class="w"> </span>id<span class="w"> </span>/<span class="w"> </span>branch<span class="w"> </span>name> |
| </pre></div> |
| </div> |
| </section> |
| <section id="download-the-build-artifacts"> |
| <h3>Download the build artifacts<a class="headerlink" href="#download-the-build-artifacts" title="Link to this heading">#</a></h3> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>artifacts<span class="w"> </span><build<span class="w"> </span>id<span class="w"> </span>/<span class="w"> </span>branch<span class="w"> </span>name> |
| </pre></div> |
| </div> |
| </section> |
| <section id="examples"> |
| <h3>Examples<a class="headerlink" href="#examples" title="Link to this heading">#</a></h3> |
| <p>Submit command accepts a list of task names and/or a list of task-group names |
| to select which tasks to build.</p> |
| <p>Run multiple builds:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>debian-stretch<span class="w"> </span>conda-linux-gcc-py37-r40 |
| <span class="go">Repository: https://github.com/kszucs/arrow@tasks</span> |
| <span class="go">Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702</span> |
| <span class="go">Version: 0.9.1.dev48+g810a7188.d20180414</span> |
| <span class="go">Pushed branches:</span> |
| <span class="go"> - debian-stretch</span> |
| <span class="go"> - conda-linux-gcc-py37-r40</span> |
| </pre></div> |
| </div> |
| <p>Just render without applying or committing the changes:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>--dry-run<span class="w"> </span>task_name |
| </pre></div> |
| </div> |
| <p>Run only <code class="docutils literal notranslate"><span class="pre">conda</span></code> package builds and a Linux one:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>--group<span class="w"> </span>conda<span class="w"> </span>centos-7 |
| </pre></div> |
| </div> |
| <p>Run <code class="docutils literal notranslate"><span class="pre">wheel</span></code> builds:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>--group<span class="w"> </span>wheel |
| </pre></div> |
| </div> |
| <p>There are multiple task groups in the <code class="docutils literal notranslate"><span class="pre">tasks.yml</span></code> like docker, integration |
| and cpp-python for running docker based tests.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">archery</span> <span class="pre">crossbow</span> <span class="pre">submit</span></code> supports multiple options and arguments, for more |
| see its help page:</p> |
| <div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>archery<span class="w"> </span>crossbow<span class="w"> </span>submit<span class="w"> </span>--help |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="archery.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Daily Development using Archery</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="../benchmarks.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Benchmarks</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <dialog id="pst-secondary-sidebar-modal"></dialog> |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#architecture">Architecture</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#executors">Executors</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#queue">Queue</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#scheduler">Scheduler</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#install">Install</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#usage">Usage</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#query-the-build-status">Query the build status</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#download-the-build-artifacts">Download the build artifacts</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#examples">Examples</a></li> |
| </ul> |
| </li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/developers/continuous_integration/crossbow.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script> |
| <script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2025 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| <!-- # L10n: Setting the PST URL as an argument as this does not need to be localized --> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.16.1. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
