Google Git
Sign in
apache / arrow-site / refs/heads/asf-site / . / docs / dev / cpp / acero / developer_guide.html
blob: 607350fd0656f2c6398eaf0b9626689ceb75ec57 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" data-content_root="" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
<title>Developer’s Guide &#8212; Apache Arrow v17.0.0.dev52</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
</script>
<!-- Loaded before other Sphinx assets -->
<link href="../../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link href="../../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link href="../../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" />
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../../_static/copybutton.css" />
<link rel="stylesheet" type="text/css" href="../../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" />
<link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css" />
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" />
<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" />
<script src="../../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script>
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
<script src="../../_static/doctools.js"></script>
<script src="../../_static/sphinx_highlight.js"></script>
<script src="../../_static/clipboard.min.js"></script>
<script src="../../_static/copybutton.js"></script>
<script src="../../_static/design-tabs.js"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'cpp/acero/developer_guide';</script>
<script>
DOCUMENTATION_OPTIONS.theme_version = '0.15.2';
DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json';
DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/';
DOCUMENTATION_OPTIONS.show_version_warning_banner = true;
</script>
<link rel="canonical" href="https://arrow.apache.org/docs/cpp/acero/developer_guide.html" />
<link rel="icon" href="../../_static/favicon.ico"/>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="Input / output and filesystems" href="../io.html" />
<link rel="prev" title="Using Acero with Substrait" href="substrait.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
/* We explicitly disable cookie tracking to avoid privacy issues */
_paq.push(['disableCookies']);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '20']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>
Back to top
</button>
<input type="checkbox"
class="sidebar-toggle"
name="__primary"
id="__primary"/>
<label class="overlay overlay-primary" for="__primary"></label>
<input type="checkbox"
class="sidebar-toggle"
name="__secondary"
id="__secondary"/>
<label class="overlay overlay-secondary" for="__secondary"></label>
<div class="search-button__wrapper">
<div class="search-button__overlay"></div>
<div class="search-button__search-container">
<form class="bd-search d-flex align-items-center"
action="../../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
id="search-input"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form></div>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar">
<div class="bd-header__inner bd-page-width">
<label class="sidebar-toggle primary-toggle" for="__primary">
<span class="fa-solid fa-bars"></span>
</label>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../../index.html">
<img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev52 - Home"/>
<script>document.write(`<img src="../../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev52 - Home"/>`);</script>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav class="navbar-nav">
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item">
<a class="nav-link nav-internal" href="../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../../developers/index.html">
Development
</a>
</li>
<li class="nav-item dropdown">
<button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links">
Implementations
</button>
<ul id="pst-nav-more-links" class="dropdown-menu">
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html">
C/GLib
</a>
</li>
<li class="nav-item current active">
<a class="nav-link dropdown-item nav-internal" href="../index.html">
C++
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md">
C#
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17">
Go
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../java/index.html">
Java
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../js/index.html">
JavaScript
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/">
Julia
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md">
MATLAB
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/">
nanoarrow
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../python/index.html">
Python
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../r/index.html">
R
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md">
Ruby
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/">
Rust
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../status.html">
Implementation Status
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/">
C++ cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/">
Java cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/">
Python cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/">
R cookbook
</a>
</li>
</ul>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<script>
document.write(`
<button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
`);
</script>
</div>
<div class="navbar-item">
<script>
document.write(`
<div class="version-switcher__container dropdown">
<button id="pst-version-switcher-button-2"
type="button"
class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-2"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-2"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-2">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div>
`);
</script></div>
<div class="navbar-item">
<script>
document.write(`
<button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
</button>
`);
</script></div>
<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">X</span></a>
</li>
</ul></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<script>
document.write(`
<button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
`);
</script>
</div>
<label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0">
<span class="fa-solid fa-outdent"></span>
</label>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<div class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav class="navbar-nav">
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item">
<a class="nav-link nav-internal" href="../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../../developers/index.html">
Development
</a>
</li>
<li class="nav-item dropdown">
<button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2">
Implementations
</button>
<ul id="pst-nav-more-links-2" class="dropdown-menu">
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html">
C/GLib
</a>
</li>
<li class="nav-item current active">
<a class="nav-link dropdown-item nav-internal" href="../index.html">
C++
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md">
C#
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17">
Go
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../java/index.html">
Java
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../js/index.html">
JavaScript
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/">
Julia
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md">
MATLAB
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/">
nanoarrow
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../python/index.html">
Python
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../r/index.html">
R
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md">
Ruby
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/">
Rust
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../status.html">
Implementation Status
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/">
C++ cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/">
Java cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/">
Python cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/">
R cookbook
</a>
</li>
</ul>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item">
<script>
document.write(`
<div class="version-switcher__container dropdown">
<button id="pst-version-switcher-button-3"
type="button"
class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-3"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-3"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-3">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div>
`);
</script></div>
<div class="navbar-item">
<script>
document.write(`
<button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
</button>
`);
</script></div>
<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">X</span></a>
</li>
</ul></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav">
<li class="toctree-l1 has-children"><a class="reference internal" href="../getting_started.html">Getting Started</a><input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../build_system.html">Using Arrow C++ in your own project</a></li>
<li class="toctree-l2"><a class="reference internal" href="../conventions.html">Conventions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/basic_arrow.html">Basic Arrow Data Structures</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/io_tutorial.html">Arrow File I/O</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/compute_tutorial.html">Arrow Compute</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/datasets_tutorial.html">Arrow Datasets</a></li>
</ul>
</li>
<li class="toctree-l1 current active has-children"><a class="reference internal" href="../user_guide.html">User Guide</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../overview.html">High-Level Overview</a></li>
<li class="toctree-l2"><a class="reference internal" href="../memory.html">Memory Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../arrays.html">Arrays</a></li>
<li class="toctree-l2"><a class="reference internal" href="../datatypes.html">Data Types</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tables.html">Tabular Data</a></li>
<li class="toctree-l2"><a class="reference internal" href="../compute.html">Compute Functions</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../gandiva.html">The Gandiva Expression Compiler</a><input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-3"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../gandiva/expr_projector_filter.html">Gandiva Expression, Projector, and Filter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../gandiva/external_func.html">Gandiva External Functions Development Guide</a></li>
</ul>
</li>
<li class="toctree-l2 current active has-children"><a class="reference internal" href="../streaming_execution.html">Acero: A C++ streaming execution engine</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-4"><i class="fa-solid fa-chevron-down"></i></label><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="overview.html">Acero Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="user_guide.html">Acero User’s Guide</a></li>
<li class="toctree-l3"><a class="reference internal" href="substrait.html">Using Acero with Substrait</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">Developer’s Guide</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../io.html">Input / output and filesystems</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ipc.html">Reading and writing the Arrow IPC format</a></li>
<li class="toctree-l2"><a class="reference internal" href="../orc.html">Reading and Writing ORC files</a></li>
<li class="toctree-l2"><a class="reference internal" href="../parquet.html">Reading and writing Parquet files</a></li>
<li class="toctree-l2"><a class="reference internal" href="../csv.html">Reading and Writing CSV files</a></li>
<li class="toctree-l2"><a class="reference internal" href="../json.html">Reading JSON files</a></li>
<li class="toctree-l2"><a class="reference internal" href="../dataset.html">Tabular Datasets</a></li>
<li class="toctree-l2"><a class="reference internal" href="../flight.html">Arrow Flight RPC</a></li>
<li class="toctree-l2"><a class="reference internal" href="../gdb.html">Debugging code using Arrow</a></li>
<li class="toctree-l2"><a class="reference internal" href="../threading.html">Thread Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../opentelemetry.html">OpenTelemetry</a></li>
<li class="toctree-l2"><a class="reference internal" href="../env_vars.html">Environment Variables</a></li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../examples/index.html">Examples</a><input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-5"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../examples/cmake_minimal_build.html">Minimal build using CMake</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/compute_and_write_example.html">Compute and Write CSV Example</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/dataset_documentation_example.html">Arrow Datasets example</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/dataset_skyhook_scan_example.html">Arrow Skyhook example</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/row_columnar_conversion.html">Row to columnar conversion</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/tuple_range_conversion.html">std::tuple-like ranges to Arrow</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/converting_recordbatch_to_tensor.html">Converting RecordBatch to Tensor</a></li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../api.html">API Reference</a><input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-6"><i class="fa-solid fa-chevron-down"></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../api/support.html">Programming Support</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/memory.html">Memory (management)</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/thread.html">Thread (management)</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/datatype.html">Data Types</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/array.html">Arrays</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/scalar.html">Scalars</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/builder.html">Array Builders</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/table.html">Two-dimensional Datasets</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/c_abi.html">C Interfaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/compute.html">Compute Functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/acero.html">Streaming Execution (Acero)</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/gandiva.html">Gandiva Expression Compiler</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/tensor.html">Tensors</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/utilities.html">Utilities</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/async.html">Asynchronous programming</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/io.html">Input / output</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/ipc.html">Arrow IPC</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/formats.html">File Formats</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/cuda.html">CUDA support</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/flight.html">Arrow Flight RPC</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/flightsql.html">Arrow Flight SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/filesystem.html">Filesystems</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/dataset.html">Dataset</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/cookbook/cpp/">C++ cookbook</a></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
</div>
<div id="rtd-footer-container"></div>
</div>
<main id="main-content" class="bd-main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../index.html" class="nav-link">C++ Implementation</a></li>
<li class="breadcrumb-item"><i class="fa-solid fa-ellipsis"></i></li>
<li class="breadcrumb-item"><a href="../streaming_execution.html" class="nav-link">Acero: A C++ streaming execution engine</a></li>
<li class="breadcrumb-item active" aria-current="page">Developer’s Guide</li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<section id="developer-s-guide">
<h1>Developer’s Guide<a class="headerlink" href="#developer-s-guide" title="Permalink to this heading">#</a></h1>
<p>This page goes into more detail into the design of Acero. It discusses how
to create custom exec nodes and describes some of the philosophies behind Acero’s
design and implementation. Finally, it gives an overview of how to extend Acero
with new behaviors and how this new behavior can be upstreamed into the core Arrow
repository.</p>
<section id="understanding-execnode">
<h2>Understanding ExecNode<a class="headerlink" href="#understanding-execnode" title="Permalink to this heading">#</a></h2>
<p>ExecNode is an abstract class with several pure virtual methods that control how the node operates:</p>
<section id="execnode-startproducing">
<h3><a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero8ExecNode14StartProducingEv" title="arrow::acero::ExecNode::StartProducing"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::StartProducing()</span></code></a><a class="headerlink" href="#execnode-startproducing" title="Permalink to this heading">#</a></h3>
<p>This method is called once at the start of the plan. Most nodes ignore this method (any
necessary initialization should happen in the constructor or Init). However, source nodes
will typically provide a custom implementation. Source nodes should schedule whatever tasks
are needed to start reading and providing the data. Source nodes are usually the primary
creator of tasks in a plan.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The ExecPlan operates on a push-based model. Sources are often pull-based. For example,
your source may be an iterator. The source node will typically then schedule tasks to pull one
item from the source and push that item into the source’s output node (via <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code>).</p>
</div>
<section id="examples">
<h4>Examples<a class="headerlink" href="#examples" title="Permalink to this heading">#</a></h4>
<ul class="simple">
<li><p>In the <code class="docutils literal notranslate"><span class="pre">table_source</span></code> node the input table is divided into batches. A task is created for
each batch and that task calls <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> on the node’s output.</p></li>
<li><p>In the <code class="docutils literal notranslate"><span class="pre">scan</span></code> node a task is created to start listing fragments from the dataset. Each listing
task then creates tasks to read batches from the fragment, asynchronously. When the batch is
full read in then a continuation schedules a new task with the exec plan. This task calls
<code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> on the scan node’s output.</p></li>
</ul>
</section>
</section>
<section id="execnode-inputreceived">
<h3><a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero8ExecNode13InputReceivedEP8ExecNode9ExecBatch" title="arrow::acero::ExecNode::InputReceived"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::InputReceived()</span></code></a><a class="headerlink" href="#execnode-inputreceived" title="Permalink to this heading">#</a></h3>
<p>This method is called many times during the execution of a plan. It is how nodes pass data
to each other. An input node will call InputReceived on its output. Acero’s execution model
is push-based. Each node pushes data into its output by calling InputReceived and passing in
a batch of data.</p>
<p>The InputReceived method is often where the actual work happens for the node. For example,
a project node will execute its expressions and create a new expanded output batch. It will then
call InputReceived on its output. InputReceived will never be called on a source node. Sink
nodes will never call InputReceived. All other nodes will experience both.</p>
<p>Some nodes (often called “pipeline breakers”) must accumulate input before they can generate
any output. For example, a sort node must accumulate all input before it can sort the data and
generate output. In these nodes the InputReceived method will typically place the data into
some kind of accumulation queue. If the node doesn’t have enough data to operate then it will
not call InputReceived. This will then be the end of the current task.</p>
<section id="id1">
<h4>Examples<a class="headerlink" href="#id1" title="Permalink to this heading">#</a></h4>
<ul class="simple">
<li><p>The <code class="docutils literal notranslate"><span class="pre">project</span></code> node runs its expressions, using the received batch as input for the expression.
A new batch is created from the input batch and the result of the expressions. The new batch is
given the same order index as the input batch and the node then calls <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> on its
output.</p></li>
<li><p>The <code class="docutils literal notranslate"><span class="pre">order_by</span></code> node inserts the batch into an accumulation queue. If this was the last batch
then the node will sort everything in the accumulation queue. The node will then call
<code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> on the output for each batch in the sorted result. A new batch index will be
assigned to each batch. Note that this final output step might also occur as a result of a call
to <code class="docutils literal notranslate"><span class="pre">InputFinished</span></code> (described below).</p></li>
</ul>
</section>
</section>
<section id="execnode-inputfinished">
<h3><a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero8ExecNode13InputFinishedEP8ExecNodei" title="arrow::acero::ExecNode::InputFinished"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::InputFinished()</span></code></a><a class="headerlink" href="#execnode-inputfinished" title="Permalink to this heading">#</a></h3>
<p>This method will be called once per input. A node will call InputFinished on its output once it
knows how many batches it will be sending to that output. Normally this happens when the node is
finished working. For example, a scan node will call InputFinished once it has finished reading
its files. However, it could call it earlier if it knows (maybe from file metadata) how many
batches will be created.</p>
<p>Some nodes will use this signal to trigger some processing. For example, a sort node need to
wait until it has received all of its input before it can sort the data. It relies on the InputFinished
call to know this has happened.</p>
<p>Even if a node is not doing any special processing when finished (e.g. a project node or filter node
doesn’t need to do any end-of-stream processing) that node will still call InputFinished on its
output.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>The InputFinished call might arrive before the final call to InputReceived. In fact, it could
even be sent out before any calls to InputReceived begin. For example, the table source node
always knows exactly how many batches it will be producing. It could choose to call InputFinished
before it ever calls InputReceived. If a node needs to do “end-of-stream” processing then it typically
uses an AtomicCounter which is a helper class to figure out when all of the data has arrived.</p>
</div>
<section id="id2">
<h4>Examples<a class="headerlink" href="#id2" title="Permalink to this heading">#</a></h4>
<ul class="simple">
<li><p>The <code class="docutils literal notranslate"><span class="pre">order_by</span></code> checks to see if it has already received all its batches. If it has then it performs
the sorting step described in the <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> example. Before it starts sending output data it
checks to see how many output batches it has (it’s possible the batch size changed as part of the
accumulating or sorting) and calls <code class="docutils literal notranslate"><span class="pre">InputFinished</span></code> on the node’s output.</p></li>
<li><p>The <code class="docutils literal notranslate"><span class="pre">fetch</span></code> node, during a call to <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> realizes it has received all the rows it was
asked for. It calls <code class="docutils literal notranslate"><span class="pre">InputFinished</span></code> on its output immediately (even though its own <code class="docutils literal notranslate"><span class="pre">InputFinished</span></code>
method has not yet been called)</p></li>
</ul>
</section>
</section>
<section id="execnode-pauseproducing-execnode-resumeproducing">
<h3><a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero8ExecNode14PauseProducingEP8ExecNode7int32_t" title="arrow::acero::ExecNode::PauseProducing"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::PauseProducing()</span></code></a> / <a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero8ExecNode15ResumeProducingEP8ExecNode7int32_t" title="arrow::acero::ExecNode::ResumeProducing"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::ResumeProducing()</span></code></a><a class="headerlink" href="#execnode-pauseproducing-execnode-resumeproducing" title="Permalink to this heading">#</a></h3>
<p>These methods control backpressure. Some nodes may need to pause their input to avoid accumulating
too much data. For example, when the user is consuming the plan with a RecordBatchReader we use a
SinkNode. The SinkNode places data in a queue that the RecordBatchReader pulls from (this is a
conversion from a push-model to a pull-model). If the user is reading the RecordBatchReader slowly then
it is possible this queue will start to fill up. For another example we can consider the write node.
This node writes data to a filesystem. If the writes are slow then data might accumulate at the
write node. As a result, the write node would need to apply backpressure.</p>
<p>When a node realizes that it needs to apply some backpressure it will call PauseProducing on its input.
Once the node has enough space to continue it will then call ResumeProducing on its input. For example,
the SinkNode would pause when its queue gets too full. As the user continues to read from the
RecordBatchReader we can expect the queue to slowly drain. Once the queue has drained enough then the
SinkNode can call ResumeProducing.</p>
<p>Source nodes typically need to provide special behavior for PauseProducing and ResumeProducing. For
example, a scan node that is reading from a file can pause reading the file. However, some source nodes
may not be able to pause in any meaningful way. There is not much point in a table source node pausing
because its data is already in memory.</p>
<p>Nodes that are neither source or sink should still forward backpressure signals. For example, when
PauseProducing is called on a project node it should call PauseProducing on its input. If a node has
multiple inputs then it should forward the signal to every input.</p>
<section id="id3">
<h4>Examples<a class="headerlink" href="#id3" title="Permalink to this heading">#</a></h4>
<ul class="simple">
<li><p>The <code class="docutils literal notranslate"><span class="pre">write</span></code> node, in its <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> method, adds a batch to a dataset writer’s queue. If the
dataset writer is then full it will return an unfinished future that will complete when it has more room.
The <code class="docutils literal notranslate"><span class="pre">write</span></code> node then calls <code class="docutils literal notranslate"><span class="pre">PauseProducing</span></code> on its input. It then adds a continuation to the future
that will call <code class="docutils literal notranslate"><span class="pre">ResumeProducing</span></code> on its input.</p></li>
<li><p>The <code class="docutils literal notranslate"><span class="pre">scan</span></code> node uses an <code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">AsyncTaskScheduler</span></code> to keep track of all the tasks it schedules. This
scheduler is throttled to limit how much concurrent I/O the <code class="docutils literal notranslate"><span class="pre">scan</span></code> node is allowed to perform. When
<code class="docutils literal notranslate"><span class="pre">PauseProducing</span></code> is called then the node will pause the scheduler. This means that any tasks queued
behind the throttle will not be submitted. However, any ongoing I/O will continue (backpressure can’t
take effect immediately). When <code class="docutils literal notranslate"><span class="pre">ResumeProducing</span></code> is called the <code class="docutils literal notranslate"><span class="pre">scan</span></code> node will unpause the scheduler.</p></li>
</ul>
</section>
</section>
<section id="execnode-stopproducing">
<h3><a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero8ExecNode13StopProducingEv" title="arrow::acero::ExecNode::StopProducing"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::StopProducing()</span></code></a><a class="headerlink" href="#execnode-stopproducing" title="Permalink to this heading">#</a></h3>
<p>StopProducing is called when a plan needs to end early. This can happen because the user cancelled
the plan and it can happen because an error occurred. Most nodes do not need to do anything here.
There is no expectation or requirement that a node sends any remaining data it has. Any node that
schedules tasks (e.g. a source node) should stop producing new data.</p>
<p>In addition to plan-wide cancellation, a node may call this method on its input if it has decided
that it has received all the data that it needs. However, because of parallelism, a node may still
receive a few calls to <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code> after it has stopped its input.</p>
<p>If any external resources are used then cleanup should happen as part of this call.</p>
<section id="id4">
<h4>Examples<a class="headerlink" href="#id4" title="Permalink to this heading">#</a></h4>
<ul class="simple">
<li><p>The <code class="docutils literal notranslate"><span class="pre">asofjoin</span></code> node has a dedicated processing thread the communicates with the main Acero threads
using a queue. When <code class="docutils literal notranslate"><span class="pre">StopProducing</span></code> is called the node inserts a poison pill into the queue. This
tells the processing thread to stop immediately. Once the processing thread stops it marks its external
task (described below) as completed which allows the plan to finish.</p></li>
<li><p>The <code class="docutils literal notranslate"><span class="pre">fetch</span></code> node, in <code class="docutils literal notranslate"><span class="pre">InputReceived</span></code>, may decide that it has all the data it needs. It can then call
<code class="docutils literal notranslate"><span class="pre">StopProducing</span></code> on its input.</p></li>
</ul>
</section>
</section>
<section id="initialization-construction-destruction">
<h3>Initialization / Construction / Destruction<a class="headerlink" href="#initialization-construction-destruction" title="Permalink to this heading">#</a></h3>
<p>Simple initialization logic (that cannot error) can be done in the constructor. If the initialization
logic may return an invalid status then it can either be done in the exec node’s factory method or
the <code class="docutils literal notranslate"><span class="pre">Init</span></code> method. The factory method is preferred for simple validation. The <code class="docutils literal notranslate"><span class="pre">Init</span></code> method is
preferred if the initialization might do expensive allocation or other resource consumption. <code class="docutils literal notranslate"><span class="pre">Init</span></code> will
always be called before <code class="docutils literal notranslate"><span class="pre">StartProducing</span></code> is called. Initialization could also be done in
<code class="docutils literal notranslate"><span class="pre">StartProducing</span></code> but keep in mind that other nodes may have started by that point.</p>
<p>In addition, there is a <code class="docutils literal notranslate"><span class="pre">Validate</span></code> method that can be overloaded to provide custom validation. This
method is normally called before <code class="docutils literal notranslate"><span class="pre">Init</span></code> but after all inputs and outputs have been added.</p>
<p>Finalization happens today in the destructor. There are a few examples today where that might be slow.
For example, in the write node, if there was an error during the plan, then we might close out some open
files here. Should there be significant finalization that is either asynchronous or could potentially
trigger an error then we could introduce a Finalize method to the ExecNode lifecycle. It hasn’t been
done yet only because it hasn’t been needed.</p>
</section>
<section id="summary">
<h3>Summary<a class="headerlink" href="#summary" title="Permalink to this heading">#</a></h3>
<table class="table" id="id5">
<caption><span class="caption-text">ExecNode Lifecycle</span><a class="headerlink" href="#id5" title="Permalink to this table">#</a></caption>
<colgroup>
<col style="width: 20.0%" />
<col style="width: 40.0%" />
<col style="width: 40.0%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>Method Name</p></th>
<th class="head"><p>This is called when…</p></th>
<th class="head"><p>A node calls this when…</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>StartProducing</p></td>
<td><p>The plan is starting</p></td>
<td><p>N/A</p></td>
</tr>
<tr class="row-odd"><td><p>InputReceived</p></td>
<td><p>Data is received from the input</p></td>
<td><p>To send data to the output</p></td>
</tr>
<tr class="row-even"><td><p>InputFinished</p></td>
<td><p>The input knows how many batches there are</p></td>
<td><p>The node can tell its output how many batches there are</p></td>
</tr>
<tr class="row-odd"><td><p>StopProducing</p></td>
<td><p>A plan is aborted or an output has enough data</p></td>
<td><p>A node has all the data it needs</p></td>
</tr>
</tbody>
</table>
</section>
</section>
<section id="extending-acero">
<h2>Extending Acero<a class="headerlink" href="#extending-acero" title="Permalink to this heading">#</a></h2>
<p>Acero instantiates a singleton <a class="reference internal" href="../api/acero.html#_CPPv4N5arrow5acero19ExecFactoryRegistryE" title="arrow::acero::ExecFactoryRegistry"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">ExecFactoryRegistry</span></code></a> which maps between names and exec node
factories (methods which create an ExecNode from options). To create a new ExecNode you can register
the node with this registry and your node will now be usable by Acero. If you would like to be able
to use this node with Substrait plans you will also need to configure the Substrait registry so that it
knows how to map Substrait to your custom node.</p>
<p>This means that you can create and add new nodes to Acero without recompiling Acero from source.</p>
</section>
<section id="scheduling-and-parallelism">
<h2>Scheduling and Parallelism<a class="headerlink" href="#scheduling-and-parallelism" title="Permalink to this heading">#</a></h2>
<p>There are many ways in that data engines can utilize multiple compute resources (e.g. multiple cores).
Before we get into the details of Acero’s scheduling we will cover a few high level topics.</p>
<section id="parallel-execution-of-plans">
<h3>Parallel Execution of Plans<a class="headerlink" href="#parallel-execution-of-plans" title="Permalink to this heading">#</a></h3>
<p>Users may want to execute multiple plans concurrently and they are welcome to do so. However, Acero has no
concept of inter-plan scheduling. Each plan will attempt to maximize its usage of compute resources and
there will likely be contention of CPU and memory and disk resources. If plans are using the default CPU &amp;
I/O thread pools this will be mitigated somewhat since they will share the same thread pool.</p>
</section>
<section id="locally-distributed-plans">
<h3>Locally Distributed Plans<a class="headerlink" href="#locally-distributed-plans" title="Permalink to this heading">#</a></h3>
<p>A common way to tackle multi-threading is to split the input into partitions and then create a plan for
each partition and then merge the results from these plans in some way. For example, let’s assume you
have 20 files and 10 cores and you want to read and sort all the data. You could create a plan for every
2 files to read and sort those files. Then you could create one extra plan that takes the input from these
10 child plans and merges the 10 input streams in a sorted fashion.</p>
<p>This approach is popular because it is how queries are distributed across multiple servers and so it
is widely supported and well understood. Acero does not do this today but there is no reason to prevent it.
Adding shuffle &amp; partition nodes to Acero should be a high priority and would enable Acero to be used by
distributed systems. Once that has been done then it should be possible to do a local shuffle (local
meaning exchanging between multiple exec plan instances on a single system) if desired.</p>
<figure class="align-default" id="id6">
<img alt="../../_images/dist_plan.svg" src="../../_images/dist_plan.svg" /><figcaption>
<p><span class="caption-text">A distributed plan can provide parallelism even if the plans themselves run serially</span><a class="headerlink" href="#id6" title="Permalink to this image">#</a></p>
</figcaption>
</figure>
</section>
<section id="pipeline-parallelism">
<h3>Pipeline Parallelism<a class="headerlink" href="#pipeline-parallelism" title="Permalink to this heading">#</a></h3>
<p>Acero attempts to maximize parallelism using pipeline parallelism. As each batch of data arrives from the
source we immediately create a task and start processing it. This means we will likely start processing
batch X before the processing of batch X-1 has completed. This is very flexible and powerful. However, it also
means that properly implementing an ExecNode is difficult.</p>
<p>For example, an ExecNode’s InputReceived method should be reentrant. In other words, it should be expected
that InputReceived will be called before the previous call to InputReceived has completed. This means that
nodes with any kind of mutable state will need mutexes or similar mechanisms to protect that state from race
conditions. It also means that tasks can easily get out of order and nodes should not expect any particular ordering
of their input (more on this later).</p>
<figure class="align-default" id="id7">
<img alt="../../_images/pipeline.svg" src="../../_images/pipeline.svg" /><figcaption>
<p><span class="caption-text">An example of pipeline parallelism on a system with 3 CPU threads and 2 I/O threads</span><a class="headerlink" href="#id7" title="Permalink to this image">#</a></p>
</figcaption>
</figure>
</section>
<section id="asynchronicity">
<h3>Asynchronicity<a class="headerlink" href="#asynchronicity" title="Permalink to this heading">#</a></h3>
<p>Some operations take a long time and may not require the CPU. Reading data from the filesystem is one example. If we
only have one thread per core then time will be wasted while we wait for these operations to complete. There
are two common solutions to this problem. A synchronous solution is often to create more threads than there are
cores with the expectation that some of them will be blocked and that is ok. This approach tends to be simpler
but it can lead to excess thread contention and requires fine-tuning.</p>
<p>Another solution is to make the slow operations asynchronous. When the slow operation starts the caller gives up
the thread and allows other tasks to run in the meantime. Once the slow operation finishes then a new task is
created to take the result and continue processing. This helps to minimize thread contention but tends to be
more complex to implement.</p>
<p>Due to a lack of standard C++ async APIs, Acero uses a combination of the two approaches. Acero has two thread pools.
The first is the CPU thread pool. This thread pool has one thread per core. Tasks in this thread pool should never
block (beyond minor delays for synchronization) and should generally be actively using CPU as much as possible. Threads
on the I/O thread pool are expected to spend most of the time idle. They should avoid doing any CPU-intensive work.
Their job is basically to wait for data to be available and schedule follow-up tasks on the CPU thread pool.</p>
<figure class="align-default" id="id8">
<img alt="../../_images/async.svg" src="../../_images/async.svg" /><figcaption>
<p><span class="caption-text">Arrow achieves asynchronous execution by combining CPU &amp; I/O thread pools</span><a class="headerlink" href="#id8" title="Permalink to this image">#</a></p>
</figcaption>
</figure>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Most nodes in Acero do not need to worry about asynchronicity. They are fully synchronous and do not spawn tasks.</p>
</div>
</section>
<section id="task-per-pipeline-and-sometimes-beyond">
<h3>Task per Pipeline (and sometimes beyond)<a class="headerlink" href="#task-per-pipeline-and-sometimes-beyond" title="Permalink to this heading">#</a></h3>
<p>An engine could choose to create a thread task for every execution of a node. However, without careful scheduling,
this leads to problems with cache locality. For example, let’s assume we have a basic plan consisting of three
exec nodes, scan, project, and then filter (this is a very common use case). Now let’s assume there are 100 batches.
In a task-per-operator model we would have tasks like “Scan Batch 5”, “Project Batch 5”, and “Filter Batch 5”. Each
of those tasks is potentially going to access the same data. For example, maybe the <cite>project</cite> and <cite>filter</cite> nodes need
to read the same column. A column which is intially created in a decode phase of the <cite>scan</cite> node. To maximize cache
utilization we would need to carefully schedule our tasks to ensure that all three of those tasks are run consecutively
and assigned to the same CPU core.</p>
<p>To avoid this problem we design tasks that run through as many nodes as possible before the task ends. This sequence
of nodes is often referred to as a “pipeline” and the nodes that end the pipeline (and thus end the task) are often
called “pipeline breakers”. Some nodes might even fall somewhere in between. For example, in a hash join node, when
we receive a batch on the probe side, and the hash table has been built, we do not need to end the task and instead keep
on running. This means that tasks might sometimes end at the join node and might sometimes continue past the join node.</p>
<figure class="align-default" id="id9">
<img alt="../../_images/pipeline_task.svg" src="../../_images/pipeline_task.svg" /><figcaption>
<p><span class="caption-text">A logical view of pipelines in a plan and two tasks, showing that pipeline boundaries may vary during a plan</span><a class="headerlink" href="#id9" title="Permalink to this image">#</a></p>
</figcaption>
</figure>
</section>
<section id="thread-pools-and-schedulers">
<h3>Thread Pools and Schedulers<a class="headerlink" href="#thread-pools-and-schedulers" title="Permalink to this heading">#</a></h3>
<p>The CPU and I/O thread pools are a part of the core Arrow-C++ library. They contain a FIFO queue of tasks and will
execute them as a thread is available. For Acero we need additional capabilities. For this we use the
AsyncTaskScheduler. In the simplest mode of operation the scheduler simply submits tasks to an underlying thread pool.
However, it is also capable of creating sub-schedulers which can apply throttling, prioritization, and task tracking:</p>
<blockquote>
<div><ul class="simple">
<li><p>A throttled scheduler associates a cost with each task. Tasks are only submitted to the underlying scheduler
if there is room. If there is not then the tasks are placed in a queue. The write node uses a throttle of size
1 to avoid reentrantly calling the dataset writer (the dataset writer does its own internal scheduling). A throttled
scheduler can be manually paused and unpaused. When paused all tasks are queued and queued tasks will not be submitted
even if there is room. This can be useful in source nodes to implement PauseProducing and ResumeProducing.</p></li>
<li><p>Priority can be applied to throttled schedulers to control the order in which queued tasks are submitted. If
there is room a task is submitted immediately (regardless of priority). However, if the throttle is full then
the task is queued and subject to prioritization. The scan node throttles how many read requests it generates
and prioritizes reading a dataset in order, if possible.</p></li>
<li><p>A task group can be used to keep track of a collection of tasks and run a finalization task when all of the
tasks have completed. This is useful for fork-join style problems. The write node uses a task group to close
a file once all outstanding write tasks for the file have completed.</p></li>
</ul>
</div></blockquote>
<p>There is research and examples out there for different ways to prioritize tasks in an execution engine. Acero has not
yet had to address this problem. Let’s go through some common situations:</p>
<blockquote>
<div><ul class="simple">
<li><p>Engines will often prioritize reading from the build side of a join node before reading from the probe side. This
would be more easily handled in Acero by applying backpressure.</p></li>
<li><p>Another common use case is to control memory accumulation. Engines will prioritize tasks which are closer to the
sink node in an effort to relieve memory pressure. However, Acero currently assumes that spilling will be added
at pipeline breakers and that memory usage in a plan will be more or less static (per core) and well below the
limits of the hardware. This might change if Acero needs to be used in an environment where there are many compute
resources and limited memory (e.g. a GPU)</p></li>
<li><p>Engines will often use work stealing algorithms to prioritize running tasks on the same core to improve cache
locality. However, since Acero uses a task-per-pipeline model there isn’t much lost opportunity for cache
parallelism that a scheduler could reclaim. Tasks only end when there is no more work that can be done with the data.</p></li>
</ul>
</div></blockquote>
<p>While there is not much prioritization in place in Acero today we do have the tools to apply it should we need to.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In addition to the AsyncTaskScheduler there is another class called the TaskScheduler. This class predates the
AsyncTaskScheduler and was designed to offer task tracking for highly efficient synchronous fork-join workloads.
If this specialized purpose meets your needs then you may consider using it. It would be interesting to profile
this against the AsyncTaskScheduler and see how closely the two compare.</p>
</div>
</section>
<section id="intra-node-parallelism">
<h3>Intra-node Parallelism<a class="headerlink" href="#intra-node-parallelism" title="Permalink to this heading">#</a></h3>
<p>Some nodes can potentially exploit parallelism within a task. For example, in the scan node we can decode
columns in parallel. In the hash join node, parallelism is sometimes exploited for complex tasks such as
building the hash table. This sort of parallelism is less common but not necessarily discouraged. Profiling should
be done first though to ensure that this extra parallelism will be helpful in your workload.</p>
</section>
<section id="all-work-happens-in-tasks">
<h3>All Work Happens in Tasks<a class="headerlink" href="#all-work-happens-in-tasks" title="Permalink to this heading">#</a></h3>
<p>All work in Acero happens as part of a task. When a plan is started the AsyncTaskScheduler is created and given an
initial task. This initial task calls StartProducing on the nodes. Tasks may schedule additional tasks. For example,
source nodes will usually schedule tasks during the call to StartProducing. Pipeline breakers will often schedule tasks
when they have accumulated all the data they need. Once all tasks in a plan are finished then the plan is considered
done.</p>
<p>Some nodes use external threads. These threads must be registered as external tasks using the BeginExternalTask method.
For example, the asof join node uses a dedicated processing thread to achieve serial execution. This dedicated thread
is registered as an external task. External tasks should be avoided where possible because they require careful
handling to avoid deadlock in error situations.</p>
</section>
</section>
<section id="ordered-execution">
<h2>Ordered Execution<a class="headerlink" href="#ordered-execution" title="Permalink to this heading">#</a></h2>
<p>Some nodes either establish an ordering to their outgoing batches or they need to be able to process batches in order.
Acero handles ordering using the <cite>batch_index</cite> property on an ExecBatch. If a node has a deterministic output order
then it should apply a batch index on batches that it emits. For example, the OrderByNode applies a new ordering to
batches (regardless of the incoming ordering). The scan node is able to attach an implicit ordering to batches which
reflects the order of the rows in the files being scanned.</p>
<p>If a node needs to process data in order then it is a bit more complicated. Because of the parallel nature of execution
we cannot guarantee that batches will arrive at a node in order. However, they can generally be expected to be “mostly
ordered”. As a result, we can insert the batches into a sequencing queue. The sequencing queue is given a callback which
is guaranteed to run on the batches, serially, in order. For example, the fetch node uses a sequencing queue. The callback
checks to see if we need to include part or all of the batch, and then slices the batch if needed.</p>
<p>Even if a node does not care about order it should try and maintain the batch index if it can. The project and filter
nodes do not care about order but they ensure that output batches keep the same index as their input batches. The filter
node will even emit empty batches if it needs to so that it can maintain the batch order without gaps.</p>
<figure class="align-default" id="id10">
<img alt="../../_images/ordered.svg" src="../../_images/ordered.svg" /><figcaption>
<p><span class="caption-text">An example of ordered execution</span><a class="headerlink" href="#id10" title="Permalink to this image">#</a></p>
</figcaption>
</figure>
</section>
<section id="partitioned-execution">
<h2>Partitioned Execution<a class="headerlink" href="#partitioned-execution" title="Permalink to this heading">#</a></h2>
<p>A stream is partitioned (or sometimes called segmented) if rows are grouped together in some way. Currently there is not
a formal notion of partitioning. However, one is starting to develop (e.g. segmented aggregation) and we may end up
introducing a more formal notion of partitions to Acero at some point as well.</p>
</section>
<section id="spillover">
<h2>Spillover<a class="headerlink" href="#spillover" title="Permalink to this heading">#</a></h2>
<p>Spillover has not yet been implemented in Acero.</p>
</section>
<section id="distributed-execution">
<h2>Distributed Execution<a class="headerlink" href="#distributed-execution" title="Permalink to this heading">#</a></h2>
<p>There are certain exec nodes which are useful when an engine is used in a distributed environment. The terminology
can vary so we will use the Substrait terminology. An exchange node sends data to different workers. Often this is
a partitioned exchange so that Acero is expected to partition each batch and distribute partitions across N different
workers. On the other end we have the capture node. This node receives data from different workers.</p>
<p>These nodes do not exist in Acero today. However, they would be in scope and we hope to have such nodes someday.</p>
</section>
<section id="profiling-tracing">
<h2>Profiling &amp; Tracing<a class="headerlink" href="#profiling-tracing" title="Permalink to this heading">#</a></h2>
<p>Acero’s tracing is currently half-implemented and there are major gaps in profiling tools. However, there has been some
effort at tracing with open telemetry and most of the necessary pieces are in place. The main thing currently lacking is
some kind of effective visualization of the tracing results.</p>
<p>In order to use the tracing that is present today you will need to build with Arrow with <cite>ARROW_WITH_OPENTELEMETRY=ON</cite>.
Then you will need to set the environment variable <cite>ARROW_TRACING_BACKEND=otlp_http</cite>. This will configure open telemetry
to export trace results (as OTLP) to the HTTP endpoint <a class="reference external" href="http://localhost:4318/v1/traces">http://localhost:4318/v1/traces</a>. You will need to configure an
open telemetry collector to collect results on that endpoint and you will need to configure a trace viewer of some kind
such as Jaeger: <a class="reference external" href="https://www.jaegertracing.io/docs/1.21/opentelemetry/">https://www.jaegertracing.io/docs/1.21/opentelemetry/</a></p>
</section>
<section id="benchmarking">
<h2>Benchmarking<a class="headerlink" href="#benchmarking" title="Permalink to this heading">#</a></h2>
<p>The most complete macro benchmarking for Acero is provided by <a class="github reference external" href="https://github.com/voltrondata-labs/arrowbench">voltrondata-labs/arrowbench</a>
These include a set of TPC-H benchmarks, executed from the R-dplyr integration, which are run on every Arrow commit and
reported to Conbench at <a class="reference external" href="https://conbench.ursa.dev/">https://conbench.ursa.dev/</a></p>
<p>In addition to these TPC-H benchmarks there are a number of micro-benchmarks for various nodes (hash-join, asof-join,
etc.) Finally, the compute functions themselves should mostly have micro-benchmarks. For more on micro benchmarks you
can refer to <a class="reference external" href="https://arrow.apache.org/docs/developers/benchmarks.html">https://arrow.apache.org/docs/developers/benchmarks.html</a></p>
<p>Any new functionality should include micro benchmarks to avoid regressions.</p>
</section>
<section id="bindings">
<h2>Bindings<a class="headerlink" href="#bindings" title="Permalink to this heading">#</a></h2>
<section id="public-api">
<h3>Public API<a class="headerlink" href="#public-api" title="Permalink to this heading">#</a></h3>
<p>The public API for Acero consists of Declaration and the various DeclarationToXyz methods. In addition the
options classes for each node are part of the public API. However, nodes are extensible and so this API is
extensible.</p>
</section>
<section id="r-dplyr">
<h3>R (dplyr)<a class="headerlink" href="#r-dplyr" title="Permalink to this heading">#</a></h3>
<p>Dplyr is an R library for programmatically building queries. The arrow-r package has dplyr bindings which
adapt the dplyr API to create Acero execution plans. In addition, there is a dplyr-substrait backend that
is in development which could eventually replace the Acero-aware binding.</p>
</section>
<section id="python">
<h3>Python<a class="headerlink" href="#python" title="Permalink to this heading">#</a></h3>
<p>The pyarrow library binds to Acero in two different ways. First, there is a direct binding in pyarrow.acero
which directly binds to the public API. Second, there are a number of compute utilities like
pyarrow.Table.group_by which uses Acero, though this is invisible to the user.</p>
</section>
<section id="java">
<h3>Java<a class="headerlink" href="#java" title="Permalink to this heading">#</a></h3>
<p>The Java implementation exposes some capabilities from Arrow datasets. These use Acero implicitly. There
are no direct bindings to Acero or Substrait in the Java implementation today.</p>
</section>
</section>
<section id="design-philosophies">
<h2>Design Philosophies<a class="headerlink" href="#design-philosophies" title="Permalink to this heading">#</a></h2>
<section id="engine-independent-compute">
<h3>Engine Independent Compute<a class="headerlink" href="#engine-independent-compute" title="Permalink to this heading">#</a></h3>
<p>If a node requires complex computation then it should encapsulate that work in abstractions that don’t depend on
any particular engine design. For example, the hash join node uses utilities such as a row encoder, a hash table,
and an exec batch builder. Other places share implementations of sequencing queues and row segmenters. The node
itself should be kept minimal and simply maps from Acero to the abstraction.</p>
<p>This helps to decouple designs from Acero’s design details and allows them to be more resilient to changes in the
engine. It also helps to promote these abstractions as capabilities on their own. Either for use in other engines
or for potential new additions to pyarrow as compute utilities.</p>
</section>
<section id="make-tasks-not-threads">
<h3>Make Tasks not Threads<a class="headerlink" href="#make-tasks-not-threads" title="Permalink to this heading">#</a></h3>
<p>If you need to run something in parallel then you should use thread tasks and not dedicated threads.</p>
<blockquote>
<div><ul class="simple">
<li><p>This keeps the thread count down (reduces thread contention and context switches)</p></li>
<li><p>This prevents deadlock (tasks get cancelled automatically in the event of a failure)</p></li>
<li><p>This simplifies profiling (Tasks can be easily measured, easier to know where all the work is)</p></li>
<li><p>This makes it possible to run without threads (sometimes users are doing their own threading and
sometimes we need to run in thread-restricted environments like emscripten)</p></li>
</ul>
</div></blockquote>
<p>Note: we do not always follow this advice currently. There is a dedicated process thread in the asof join
node. Dedicated threads are “ok” for experimental use but we’d like to migrate away from them.</p>
</section>
<section id="don-t-block-on-cpu-threads">
<h3>Don’t Block on CPU Threads<a class="headerlink" href="#don-t-block-on-cpu-threads" title="Permalink to this heading">#</a></h3>
<p>If you need to run a potentially long running activity that is not actively using CPU resources (e.g. reading from
disk, network I/O, waiting on an external library using its own threads) then you should use asynchronous utilities
to ensure that you do not block CPU threads.</p>
</section>
<section id="don-t-reinvent-the-wheel">
<h3>Don’t Reinvent the Wheel<a class="headerlink" href="#don-t-reinvent-the-wheel" title="Permalink to this heading">#</a></h3>
<p>Each node should not be a standalone island of utilities. Where possible, computation should be pushed
either into compute functions or into common shared utilities. This is the only way a project as large as
this can hope to be maintained.</p>
</section>
<section id="avoid-query-optimization">
<h3>Avoid Query Optimization<a class="headerlink" href="#avoid-query-optimization" title="Permalink to this heading">#</a></h3>
<p>Writing an efficient Acero plan can be challenging. For example, filter expressions and column selection
should be pushed down into the scan node so that the data isn’t read from disk. Expressions should be
simplified and common sub-expressions factored out. The build side of a hash join node should be the
smaller of the two inputs.</p>
<p>However, figuring these problems out is a challenge reserved for a query planner or a query optimizer.
Creating a query optimizer is a challenging task beyond the scope of Acero. With adoption of Substrait
we hope utilities will eventually emerge that solve these problems. As a result, we generally avoid doing
any kind of query optimization within Acero. Acero should interpret declarations as literally as possible.
This helps reduce maintenance and avoids surprises.</p>
<p>We also realize that this is not always possible. For example, the hash join node currently detects if there
is a chain of hash join operators and, if there is, it configure bloom filters between the operators. This is
technically a task that could be left to a query optimizer. However, this behavior is rather specific to Acero
and fairly niche and so it is unlikely it will be introduced to an optimizer anytime soon.</p>
</section>
</section>
<section id="performance-guidelines">
<h2>Performance Guidelines<a class="headerlink" href="#performance-guidelines" title="Permalink to this heading">#</a></h2>
<section id="batch-size">
<h3>Batch Size<a class="headerlink" href="#batch-size" title="Permalink to this heading">#</a></h3>
<p>Perhaps the most discussed performance criteria is batch size. Acero was originally
designed based on research to follow a morsel-batch model. Tasks are created based on
a large batch of rows (a morsel). The goal is for the morsel to be large enough to justify
the overhead of a task. Within a task the data is further subdivided into batches.
Each batch should be small enough to fit comfortable into CPU cache (often the L2 cache).</p>
<p>This sets up two loops. The outer loop is parallel and the inner loop is not:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="n">morsel</span> <span class="ow">in</span> <span class="n">dataset</span><span class="p">:</span> <span class="c1"># parallel</span>
<span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">morsel</span><span class="p">:</span>
<span class="n">run_pipeline</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>
</pre></div>
</div>
<p>The advantage of this style of execution is that successive nodes (or successive operations
within an exec node) that access the same column are likely to benefit from cache. It also
is essential for functions that require random access to data. It maximizes parallelism while
minimizing the data transfer from main memory to CPU cache.</p>
<figure class="align-default" id="id11">
<img alt="../../_images/microbatch.svg" src="../../_images/microbatch.svg" /><figcaption>
<p><span class="caption-text">If multiple passes through the data are needed (or random access) and the batch is much bigger
then the cache then performance suffers. Breaking the task into smaller batches helps improve
task locality.</span><a class="headerlink" href="#id11" title="Permalink to this image">#</a></p>
</figcaption>
</figure>
<p>The morsel/batch model is reflected in a few places in Acero:</p>
<blockquote>
<div><ul class="simple">
<li><p>In most source nodes we will try and grab batches of 1Mi rows. This is often configurable.</p></li>
<li><p>In the source node we then iterate and slice off batches of 32Ki rows. This is not currently
configurable.</p></li>
<li><p>The hash join node currently requires that a batches contain at 32Ki rows or less as it uses
16-bit signed integers as row indices in some places.</p></li>
</ul>
</div></blockquote>
<p>However, this guidance is debateable. Profiling has shown that we do not get any real benefit
from moving to a smaller batch size. It seems any advantage we do get is lost in per-batch
overhead. Most of this overhead appears to be due to various per-batch allocations. In addition,
depending on your hardware, it’s not clear that CPU Cache&lt;-&gt;RAM will always be the bottleneck. A
combination of linear access, pre-fetch, and high CPU&lt;-&gt;RAM bandwidth can alleviate the penalty
of cache misses.</p>
<p>As a result, this section is included in the guide to provide historical context, but should not
be considered binding.</p>
</section>
</section>
<section id="ongoing-deprecated-work">
<h2>Ongoing &amp; Deprecated Work<a class="headerlink" href="#ongoing-deprecated-work" title="Permalink to this heading">#</a></h2>
<p>The following efforts are ongoing. They are described here to explain certain duplication in the
code base as well as explain types that are going away.</p>
<section id="scanner-v2">
<h3>Scanner v2<a class="headerlink" href="#scanner-v2" title="Permalink to this heading">#</a></h3>
<p>The scanner is currently a node in the datasets module registered with the factory registry as “scan”.
This node was written prior to Acero and made extensive use of AsyncGenerator to scan multiple files
in parallel. Unfortunately, the use of AsyncGenerator made the scan difficult to profile, difficult
to debug, and impossible to cancel. A new scan node is in progress. It is currently registered with
the name “scan2”. The new scan node uses the AsyncTaskScheduler instead of AsyncGenerator and should
provide additional features such as the ability to skip rows and handle nested column projection (for
formats that support it)</p>
</section>
<section id="orderbysink-and-selectksink">
<h3>OrderBySink and SelectKSink<a class="headerlink" href="#orderbysink-and-selectksink" title="Permalink to this heading">#</a></h3>
<p>These two exec nodes provided custom sink implementations. They were written before ordered execution
was added to Acero and were the only way to generate ordered output. However, they had to be placed
at the end of a plan and the fact that they were custom sink nodes made them difficult to describe with
Declaration. The OrderByNode and FetchNode replace these. These are kept at the moment until existing
bindings move away from them.</p>
</section>
</section>
<section id="upstreaming-changes">
<h2>Upstreaming Changes<a class="headerlink" href="#upstreaming-changes" title="Permalink to this heading">#</a></h2>
<p>Acero is designed so that it can be extended without recompilation. You can easily add new compute
functions and exec nodes without creating a fork or compiling Acero. However, as you develop new
features that are generally useful, we hope you will make time to upstream your changes.</p>
<p>Even though we welcome these changes we have to admit that there is a cost to this process. Upstreaming
code requires that the new module behave correctly, but that is typically the easier part to review.
More importantly, upstreaming code is a process of transferring the maintenance burden from yourself to
the wider Arrow C++ project maintainers. This requires a deep understanding of the code by maintainers,
it requires the code be consistent with the style of the project, and it requires that the code be well
tested with unit tests to aid in regression.</p>
<p>Because of this, we highly recommend taking the following steps:</p>
<ul class="simple">
<li><p>As you are starting out you should send a message to the mailing list announcing your intentions and
design. This will help you determine if there is wider interest in the feature and others may have
ideas or suggestions to contribute early on in the process.</p>
<ul>
<li><p>If there is not much interest in the feature then keep in mind that it may be difficult to eventually
upstream the change. The maintenance capacity of the team is limited and we try and prioritize
features that are in high demand.</p></li>
</ul>
</li>
<li><p>We recommend developing and testing the change on your own fork until you get it to a point where you
are fairly confident things are working correctly. If the change is large then you might also think
about how you can break up the change into smaller pieces. As you do this you can share both the larger
PR (as a draft PR or a branch on your local fork) and the smaller PRs. This way we can see the context
of the smaller PRs. However, if you do break things up, smaller PRs should still ideally stand on their
own.</p></li>
<li><p>Any PR will need to have the following:</p>
<ul>
<li><p>Unit tests converting the new functionality</p></li>
<li><p>Microbenchmarks if there is any significant compute work going on</p></li>
<li><p>Examples demonstrating how to use the new feature</p></li>
<li><p>Updates to the API reference and this guide</p></li>
<li><p>Passing CI (you can enable GitHub Actions on your fork and that will allow most CI jobs to run before
you create your PR)</p></li>
</ul>
</li>
</ul>
</section>
</section>
</article>
<footer class="prev-next-footer">
<div class="prev-next-area">
<a class="left-prev"
href="substrait.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Using Acero with Substrait</p>
</div>
</a>
<a class="right-next"
href="../io.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">Input / output and filesystems</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#understanding-execnode">Understanding ExecNode</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execnode-startproducing"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::StartProducing()</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#examples">Examples</a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execnode-inputreceived"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::InputReceived()</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#id1">Examples</a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execnode-inputfinished"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::InputFinished()</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#id2">Examples</a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execnode-pauseproducing-execnode-resumeproducing"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::PauseProducing()</span></code> / <code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::ResumeProducing()</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#id3">Examples</a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#execnode-stopproducing"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ExecNode::StopProducing()</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#id4">Examples</a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#initialization-construction-destruction">Initialization / Construction / Destruction</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#summary">Summary</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#extending-acero">Extending Acero</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#scheduling-and-parallelism">Scheduling and Parallelism</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#parallel-execution-of-plans">Parallel Execution of Plans</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#locally-distributed-plans">Locally Distributed Plans</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#pipeline-parallelism">Pipeline Parallelism</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#asynchronicity">Asynchronicity</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#task-per-pipeline-and-sometimes-beyond">Task per Pipeline (and sometimes beyond)</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#thread-pools-and-schedulers">Thread Pools and Schedulers</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#intra-node-parallelism">Intra-node Parallelism</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#all-work-happens-in-tasks">All Work Happens in Tasks</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ordered-execution">Ordered Execution</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#partitioned-execution">Partitioned Execution</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#spillover">Spillover</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#distributed-execution">Distributed Execution</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#profiling-tracing">Profiling &amp; Tracing</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#benchmarking">Benchmarking</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#bindings">Bindings</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#public-api">Public API</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#r-dplyr">R (dplyr)</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#python">Python</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#java">Java</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#design-philosophies">Design Philosophies</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#engine-independent-compute">Engine Independent Compute</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#make-tasks-not-threads">Make Tasks not Threads</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#don-t-block-on-cpu-threads">Don’t Block on CPU Threads</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#don-t-reinvent-the-wheel">Don’t Reinvent the Wheel</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#avoid-query-optimization">Avoid Query Optimization</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#performance-guidelines">Performance Guidelines</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#batch-size">Batch Size</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ongoing-deprecated-work">Ongoing &amp; Deprecated Work</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#scanner-v2">Scanner v2</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#orderbysink-and-selectksink">OrderBySink and SelectKSink</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#upstreaming-changes">Upstreaming Changes</a></li>
</ul>
</nav></div>
<div class="sidebar-secondary-item">
<div class="tocsection editthispage">
<a href="https://github.com/apache/arrow/edit/main/docs/source/cpp/acero/developer_guide.rst">
<i class="fa-solid fa-pencil"></i>
Edit on GitHub
</a>
</div>
</div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script src="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script>
<script src="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 2016-2024 Apache Software Foundation.
Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2.
</p></div>
</div>
</div>
</footer>
</body>
</html>