| |
| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" /> |
| |
| <title>Acero Overview — Apache Arrow v17.0.0.dev52</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; |
| </script> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| |
| |
| <link href="../../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> |
| |
| <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> |
| <link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> |
| <script src="../../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script> |
| <script src="../../_static/doctools.js"></script> |
| <script src="../../_static/sphinx_highlight.js"></script> |
| <script src="../../_static/clipboard.min.js"></script> |
| <script src="../../_static/copybutton.js"></script> |
| <script src="../../_static/design-tabs.js"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'cpp/acero/overview';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.15.2'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/cpp/acero/overview.html" /> |
| <link rel="icon" href="../../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../../genindex.html" /> |
| <link rel="search" title="Search" href="../../search.html" /> |
| <link rel="next" title="Acero User’s Guide" href="user_guide.html" /> |
| <link rel="prev" title="Acero: A C++ streaming execution engine" href="../streaming_execution.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i> |
| Back to top |
| </button> |
| |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__primary" |
| id="__primary"/> |
| <label class="overlay overlay-primary" for="__primary"></label> |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__secondary" |
| id="__secondary"/> |
| <label class="overlay overlay-secondary" for="__secondary"></label> |
| |
| <div class="search-button__wrapper"> |
| <div class="search-button__overlay"></div> |
| <div class="search-button__search-container"> |
| <form class="bd-search d-flex align-items-center" |
| action="../../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| id="search-input" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form></div> |
| </div> |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar"> |
| <div class="bd-header__inner bd-page-width"> |
| <label class="sidebar-toggle primary-toggle" for="__primary"> |
| <span class="fa-solid fa-bars"></span> |
| </label> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| <a class="navbar-brand logo" href="../../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev52 - Home"/> |
| <script>document.write(`<img src="../../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev52 - Home"/>`);</script> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link dropdown-item nav-internal" href="../index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| |
| <label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0"> |
| <span class="fa-solid fa-outdent"></span> |
| </label> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <div class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links-2" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link dropdown-item nav-internal" href="../index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../getting_started.html">Getting Started</a><input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../build_system.html">Using Arrow C++ in your own project</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../conventions.html">Conventions</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../tutorials/basic_arrow.html">Basic Arrow Data Structures</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../tutorials/io_tutorial.html">Arrow File I/O</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../tutorials/compute_tutorial.html">Arrow Compute</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../tutorials/datasets_tutorial.html">Arrow Datasets</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1 current active has-children"><a class="reference internal" href="../user_guide.html">User Guide</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul class="current"> |
| <li class="toctree-l2"><a class="reference internal" href="../overview.html">High-Level Overview</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../memory.html">Memory Management</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../arrays.html">Arrays</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../datatypes.html">Data Types</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../tables.html">Tabular Data</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../compute.html">Compute Functions</a></li> |
| <li class="toctree-l2 has-children"><a class="reference internal" href="../gandiva.html">The Gandiva Expression Compiler</a><input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-3"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l3"><a class="reference internal" href="../gandiva/expr_projector_filter.html">Gandiva Expression, Projector, and Filter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="../gandiva/external_func.html">Gandiva External Functions Development Guide</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2 current active has-children"><a class="reference internal" href="../streaming_execution.html">Acero: A C++ streaming execution engine</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-4"><i class="fa-solid fa-chevron-down"></i></label><ul class="current"> |
| <li class="toctree-l3 current active"><a class="current reference internal" href="#">Acero Overview</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="user_guide.html">Acero User’s Guide</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="substrait.html">Using Acero with Substrait</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="developer_guide.html">Developer’s Guide</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="../io.html">Input / output and filesystems</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../ipc.html">Reading and writing the Arrow IPC format</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../orc.html">Reading and Writing ORC files</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../parquet.html">Reading and writing Parquet files</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../csv.html">Reading and Writing CSV files</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../json.html">Reading JSON files</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../dataset.html">Tabular Datasets</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../gdb.html">Debugging code using Arrow</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../threading.html">Thread Management</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../opentelemetry.html">OpenTelemetry</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../env_vars.html">Environment Variables</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../examples/index.html">Examples</a><input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-5"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../examples/cmake_minimal_build.html">Minimal build using CMake</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../examples/compute_and_write_example.html">Compute and Write CSV Example</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../examples/dataset_documentation_example.html">Arrow Datasets example</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../examples/dataset_skyhook_scan_example.html">Arrow Skyhook example</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../examples/row_columnar_conversion.html">Row to columnar conversion</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../examples/tuple_range_conversion.html">std::tuple-like ranges to Arrow</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../examples/converting_recordbatch_to_tensor.html">Converting RecordBatch to Tensor</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="../api.html">API Reference</a><input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-6"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="../api/support.html">Programming Support</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/memory.html">Memory (management)</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/thread.html">Thread (management)</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/datatype.html">Data Types</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/array.html">Arrays</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/scalar.html">Scalars</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/builder.html">Array Builders</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/table.html">Two-dimensional Datasets</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/c_abi.html">C Interfaces</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/compute.html">Compute Functions</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/acero.html">Streaming Execution (Acero)</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/gandiva.html">Gandiva Expression Compiler</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/tensor.html">Tensors</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/utilities.html">Utilities</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/async.html">Asynchronous programming</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/io.html">Input / output</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/ipc.html">Arrow IPC</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/formats.html">File Formats</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/cuda.html">CUDA support</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/flightsql.html">Arrow Flight SQL</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/filesystem.html">Filesystems</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="../api/dataset.html">Dataset</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/cookbook/cpp/">C++ cookbook</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| </div> |
| |
| <div id="rtd-footer-container"></div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| |
| |
| |
| |
| <nav aria-label="Breadcrumb"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">C++ Implementation</a></li> |
| |
| |
| <li class="breadcrumb-item"><i class="fa-solid fa-ellipsis"></i></li> |
| |
| |
| <li class="breadcrumb-item"><a href="../streaming_execution.html" class="nav-link">Acero: A C++ streaming execution engine</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page">Acero Overview</li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="acero-overview"> |
| <h1>Acero Overview<a class="headerlink" href="#acero-overview" title="Permalink to this heading">#</a></h1> |
| <p>This page gives an overview of the basic Acero concepts and helps distinguish Acero |
| from other modules in the Arrow code base. It’s intended for users, developers, |
| potential contributors, and for those that would like to extend Acero, either for |
| research or for business use. This page assumes the reader is already familiar with |
| core Arrow concepts. This page does not expect any existing knowledge in relational |
| algebra.</p> |
| <section id="what-is-acero"> |
| <h2>What is Acero?<a class="headerlink" href="#what-is-acero" title="Permalink to this heading">#</a></h2> |
| <p>Acero is a C++ library that can be used to analyze large (potentially infinite) streams |
| of data. Acero allows computation to be expressed as an “execution plan” (<a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero8ExecPlanE" title="arrow::acero::ExecPlan"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">ExecPlan</span></code></a>). |
| An execution plan takes in zero or more streams of input data and emits a single |
| stream of output data. The plan describes how the data will be transformed as it |
| passes through. For example, a plan might:</p> |
| <ul class="simple"> |
| <li><p>Merge two streams of data using a common column</p></li> |
| <li><p>Create additional columns by evaluating expressions against the existing columns</p></li> |
| <li><p>Consume a stream of data by writing it to disk in a partitioned layout</p></li> |
| </ul> |
| <img alt="A sample execution plan that joins three streams of data and writes to disk" class="align-center" src="../../_images/simple_graph.svg" /><section id="acero-is-not"> |
| <h3>Acero is not…<a class="headerlink" href="#acero-is-not" title="Permalink to this heading">#</a></h3> |
| <section id="a-library-for-data-scientists"> |
| <h4>A Library for Data Scientists<a class="headerlink" href="#a-library-for-data-scientists" title="Permalink to this heading">#</a></h4> |
| <p>Acero is not intended to be used directly by data scientists. It is expected that |
| end users will typically be using some kind of frontend. For example, Pandas, Ibis, |
| or SQL. The API for Acero is focused around capabilities and available algorithms. |
| However, such users may be interested in knowing more about how Acero works so that |
| they can better understand how the backend processing for their libraries operates.</p> |
| </section> |
| <section id="a-database"> |
| <h4>A Database<a class="headerlink" href="#a-database" title="Permalink to this heading">#</a></h4> |
| <p>A database (or DBMS) is typically a much more expansive application and often packaged |
| as a standalone service. Acero could be a component in a database (most databases have |
| some kind of execution engine) or could be a component in some other data processing |
| application that hardly resembles a database. Acero does not concern itself with |
| user management, external communication, isolation, durability, or consistency. In |
| addition, Acero is focused primarily on the read path, and the write utilities lack |
| any sort of transaction support.</p> |
| </section> |
| <section id="an-optimizer"> |
| <h4>An Optimizer<a class="headerlink" href="#an-optimizer" title="Permalink to this heading">#</a></h4> |
| <p>Acero does not have an SQL parser. It does not have a query planner. It does not have |
| any sort of optimizer. Acero expects to be given very detailed and low-level instructions |
| on how to manipulate data and then it will perform that manipulation exactly as described.</p> |
| <p>Creating the best execution plan is very hard. Small details can have a big impact on |
| performance. We do think an optimizer is important but we believe it should be |
| implemented independent of acero, hopefully in a composable way through standards such |
| as Substrait so that any backend could leverage it.</p> |
| </section> |
| <section id="distributed"> |
| <h4>Distributed<a class="headerlink" href="#distributed" title="Permalink to this heading">#</a></h4> |
| <p>Acero does not provide distributed execution. However, Acero aims to be usable by a distributed |
| query execution engine. In other words, Acero will not configure and coordinate workers but |
| it does expect to be used as a worker. Sometimes, the distinction is a bit fuzzy. For example, |
| an Acero source may be a smart storage device that is capable of performing filtering or other |
| advanced analytics. One might consider this a distributed plan. The key distinction is Acero |
| does not have the capability of transforming a logical plan into a distributed execution plan. |
| That step will need to be done elsewhere.</p> |
| </section> |
| </section> |
| <section id="acero-vs"> |
| <h3>Acero vs…<a class="headerlink" href="#acero-vs" title="Permalink to this heading">#</a></h3> |
| <section id="arrow-compute"> |
| <h4>Arrow Compute<a class="headerlink" href="#arrow-compute" title="Permalink to this heading">#</a></h4> |
| <p>This is described in more detail in <a class="reference internal" href="#cpp-vs-acero"><span class="std std-ref">Relation to Arrow C++</span></a> but the key difference |
| is that Acero handles streams of data and Arrow Compute handles situations where all the |
| data is in memory.</p> |
| </section> |
| <section id="arrow-datasets"> |
| <h4>Arrow Datasets<a class="headerlink" href="#arrow-datasets" title="Permalink to this heading">#</a></h4> |
| <p>The Arrow datasets library provides some basic routines for discovering, scanning, and |
| writing collections of files. The datasets module depends on Acero. Both scanning and |
| writing datasets uses Acero. The scan node and the write node are part of the datasets |
| module. This helps to keep the complexity of file formats and filesystems out of the core |
| Acero logic.</p> |
| </section> |
| <section id="substrait"> |
| <h4>Substrait<a class="headerlink" href="#substrait" title="Permalink to this heading">#</a></h4> |
| <p>Substrait is a project establishing standards for query plans. Acero executes query plans |
| and generates data. This makes Acero a Substrait consumer. There are more details on the |
| Substrait capabilities in <a class="reference internal" href="substrait.html#acero-substrait"><span class="std std-ref">Using Acero with Substrait</span></a>.</p> |
| </section> |
| <section id="datafusion-duckdb-velox-etc"> |
| <h4>Datafusion / DuckDb / Velox / Etc.<a class="headerlink" href="#datafusion-duckdb-velox-etc" title="Permalink to this heading">#</a></h4> |
| <p>There are many columnar data engines emerging. We view this as a good thing and encourage |
| projects like Substrait to help allow switching between engines as needed. We generally |
| discourage comparative benchmarks as they are almost inevitably going to be workload-driven |
| and rarely manage to capture an apples-vs-apples comparison. Discussions of the pros and |
| cons of each is beyond the scope of this guide.</p> |
| </section> |
| </section> |
| </section> |
| <section id="relation-to-arrow-c"> |
| <span id="cpp-vs-acero"></span><h2>Relation to Arrow C++<a class="headerlink" href="#relation-to-arrow-c" title="Permalink to this heading">#</a></h2> |
| <p>The Acero module is part of the Arrow C++ implementation. It is built as a separate |
| module but it depends on core Arrow modules and does not stand alone. Acero uses |
| and extends the capabilities from the core Arrow module and the Arrow compute kernels.</p> |
| <img alt="A diagram of layers with core on the left, compute in the middle, and acero on the right" src="../../_images/layers.svg" /><p>The core Arrow library provides containers for buffers and arrays that are laid out according |
| to the Arrow columnar format. With few exceptions the core Arrow library does not examine |
| or modify the contents of buffers. For example, converting a string array from lowercase |
| strings to uppercase strings would not be a part of the core Arrow library because that would |
| require examining the contents of the array.</p> |
| <p>The compute module expands on the core library and provides functions which analyze and |
| transform data. The compute module’s capabilities are all exposed via a function registry. |
| An Arrow “function” accepts zero or more arrays, batches, or tables, and produces an array, |
| batch, or table. In addition, function calls can be combined, along with field references |
| and literals, to form an expression (a tree of function calls) which the compute module can |
| evaluate. For example, calculating <code class="docutils literal notranslate"><span class="pre">x</span> <span class="pre">+</span> <span class="pre">(y</span> <span class="pre">*</span> <span class="pre">3)</span></code> given a table with columns <code class="docutils literal notranslate"><span class="pre">x</span></code> and <code class="docutils literal notranslate"><span class="pre">y</span></code>.</p> |
| <img alt="A sample expression tree" class="align-center" src="../../_images/expression_ast.svg" /><p>Acero expands on these capabilities by adding compute operations for streams of data. For |
| example, a project node can apply a compute expression on a stream of batches. This will |
| create a new stream of batches with the result of the expression added as a new column. These |
| nodes can be combined into a graph to form a more complex execution plan. This is very similar |
| to the way functions are combined into a tree to form a complex expression.</p> |
| <img alt="A simple plan that uses compute expressions" src="../../_images/simple_plan.svg" /><div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Acero does not use the <a class="reference internal" href="../api/table.html#_CPPv4N5arrow5TableE" title="arrow::Table"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::Table</span></code></a> or <a class="reference internal" href="../api/array.html#_CPPv4N5arrow12ChunkedArrayE" title="arrow::ChunkedArray"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::ChunkedArray</span></code></a> containers |
| from the core Arrow library. This is because Acero operates on streams of batches and |
| so there is no need for a multi-batch container of data. This helps to reduce the |
| complexity of Acero and avoids tricky situations that can arise from tables whose |
| columns have different chunk sizes. Acero will often use <a class="reference internal" href="../api/compute.html#_CPPv4N5arrow5DatumE" title="arrow::Datum"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::Datum</span></code></a> |
| which is a variant from the core module that can hold many different types. Within |
| Acero, a datum will always hold either an <a class="reference internal" href="../api/array.html#_CPPv4N5arrow5ArrayE" title="arrow::Array"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::Array</span></code></a> or a <a class="reference internal" href="../api/scalar.html#_CPPv4N5arrow6ScalarE" title="arrow::Scalar"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::Scalar</span></code></a>.</p> |
| </div> |
| </section> |
| <section id="core-concepts"> |
| <h2>Core Concepts<a class="headerlink" href="#core-concepts" title="Permalink to this heading">#</a></h2> |
| <section id="execnode"> |
| <h3>ExecNode<a class="headerlink" href="#execnode" title="Permalink to this heading">#</a></h3> |
| <p>The most basic concept in Acero is the ExecNode. An ExecNode has zero or more inputs and |
| zero or one outputs. If an ExecNode has zero inputs we call it a source and if an ExecNode |
| does not have an output then we call it a sink. There are many different kinds of nodes and |
| each one transforms its inputs in different ways. For example:</p> |
| <ul class="simple"> |
| <li><p>A scan node is a source node that reads data from files</p></li> |
| <li><p>An aggregate node accumulates batches of data to compute summary statistics</p></li> |
| <li><p>A filter node removes rows from the data according to a filter expression</p></li> |
| <li><p>A table sink node accumulates data into a table</p></li> |
| </ul> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>A full list of the available compute modules is included in the <a class="reference internal" href="user_guide.html#execnode-list"><span class="std std-ref">user’s guide</span></a></p> |
| </div> |
| </section> |
| <section id="execbatch"> |
| <span id="exec-batch"></span><h3>ExecBatch<a class="headerlink" href="#execbatch" title="Permalink to this heading">#</a></h3> |
| <p>Batches of data are represented by the ExecBatch class. An ExecBatch is a 2D structure that |
| is very similar to a RecordBatch. It can have zero or more columns and all of the columns |
| must have the same length. There are a few key differences from ExecBatch:</p> |
| <figure class="align-default" id="id1"> |
| <img alt="../../_images/rb_vs_eb.svg" src="../../_images/rb_vs_eb.svg" /><figcaption> |
| <p><span class="caption-text">Both the record batch and the exec batch have strong ownership of the arrays & buffers</span><a class="headerlink" href="#id1" title="Permalink to this image">#</a></p> |
| </figcaption> |
| </figure> |
| <ul class="simple"> |
| <li><p>An <cite>ExecBatch</cite> does not have a schema. This is because an <cite>ExecBatch</cite> is assumed to be |
| part of a stream of batches and the stream is assumed to have a consistent schema. So |
| the schema for an <cite>ExecBatch</cite> is typically stored in the ExecNode.</p></li> |
| <li><p>Columns in an <cite>ExecBatch</cite> are either an <cite>Array</cite> or a <cite>Scalar</cite>. When a column is a <cite>Scalar</cite> |
| this means that the column has a single value for every row in the batch. An <cite>ExecBatch</cite> |
| also has a length property which describes how many rows are in a batch. So another way to |
| view a <cite>Scalar</cite> is a constant array with <cite>length</cite> elements.</p></li> |
| <li><p>An <cite>ExecBatch</cite> contains additional information used by the exec plan. For example, an |
| <cite>index</cite> can be used to describe a batch’s position in an ordered stream. We expect |
| that <cite>ExecBatch</cite> will also evolve to contain additional fields such as a selection vector.</p></li> |
| </ul> |
| <figure class="align-default" id="id2"> |
| <img alt="../../_images/scalar_vs_array.svg" src="../../_images/scalar_vs_array.svg" /><figcaption> |
| <p><span class="caption-text">There are four different ways to represent the given batch of data using different combinations |
| of arrays and scalars. All four exec batches should be considered semantically equivalent.</span><a class="headerlink" href="#id2" title="Permalink to this image">#</a></p> |
| </figcaption> |
| </figure> |
| <p>Converting from a record batch to an exec batch is always zero copy. Both RecordBatch and ExecBatch |
| refer to the exact same underlying arrays. Converting from an exec batch to a record batch is |
| only zero copy if there are no scalars in the exec batch.</p> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Both Acero and the compute module have “lightweight” versions of batches and arrays. |
| In the compute module these are called <cite>BatchSpan</cite>, <cite>ArraySpan</cite>, and <cite>BufferSpan</cite>. In |
| Acero the concept is called <cite>KeyColumnArray</cite>. These types were developed concurrently |
| and serve the same purpose. They aim to provide an array container that can be completely |
| stack allocated (provided the data type is non-nested) in order to avoid heap allocation |
| overhead. Ideally these two concepts will be merged someday.</p> |
| </div> |
| </section> |
| <section id="execplan"> |
| <h3>ExecPlan<a class="headerlink" href="#execplan" title="Permalink to this heading">#</a></h3> |
| <p>An ExecPlan represents a graph of ExecNode objects. A valid ExecPlan must always have at |
| least one source node but it does not technically need to have a sink node. The ExecPlan contains |
| resources shared by all of the nodes and has utility functions to control starting and stopping |
| execution of the nodes. Both ExecPlan and ExecNode are tied to the lifecycle of a single execution. |
| They have state and are not expected to be restartable.</p> |
| <div class="admonition warning"> |
| <p class="admonition-title">Warning</p> |
| <p>The structures within Acero, including <cite>ExecBatch</cite>, are still experimental. The <cite>ExecBatch</cite> |
| class should not be used outside of Acero. Instead, an <cite>ExecBatch</cite> should be converted to |
| a more standard structure such as a <cite>RecordBatch</cite>.</p> |
| <p>Similarly, an ExecPlan is an internal concept. Users creating plans should be using Declaration |
| objects. APIs for consuming and executing plans should abstract away the details of the underlying |
| plan and not expose the object itself.</p> |
| </div> |
| </section> |
| <section id="declaration"> |
| <h3>Declaration<a class="headerlink" href="#declaration" title="Permalink to this heading">#</a></h3> |
| <p>A Declaration is a blueprint for an ExecNode. Declarations can be combined into a graph to |
| form the blueprint for an ExecPlan. A Declaration describes the computation that needs to be |
| done but is not actually responsible for carrying out the computation. In this way, a Declaration is |
| analogous to an expression. It is expected that Declarations will need to be converted to and from |
| various query representations (e.g. Substrait). The Declaration objects are the public API, combined |
| with the DeclarationToXyz methods, are the current public API for Acero.</p> |
| <figure class="align-default" id="id3"> |
| <img alt="../../_images/decl_vs_ep.svg" src="../../_images/decl_vs_ep.svg" /><figcaption> |
| <p><span class="caption-text">A declaration is a blueprint that is used to instantiate exec plan instances</span><a class="headerlink" href="#id3" title="Permalink to this image">#</a></p> |
| </figcaption> |
| </figure> |
| </section> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="../streaming_execution.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Acero: A C++ streaming execution engine</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="user_guide.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Acero User’s Guide</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-is-acero">What is Acero?</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#acero-is-not">Acero is not…</a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#a-library-for-data-scientists">A Library for Data Scientists</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#a-database">A Database</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#an-optimizer">An Optimizer</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#distributed">Distributed</a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#acero-vs">Acero vs…</a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#arrow-compute">Arrow Compute</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#arrow-datasets">Arrow Datasets</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#substrait">Substrait</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion-duckdb-velox-etc">Datafusion / DuckDb / Velox / Etc.</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#relation-to-arrow-c">Relation to Arrow C++</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#core-concepts">Core Concepts</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execnode">ExecNode</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execbatch">ExecBatch</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execplan">ExecPlan</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#declaration">Declaration</a></li> |
| </ul> |
| </li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/cpp/acero/overview.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script> |
| <script src="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2024 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
