blob: fec6b3e9f54503b3a11d3bd697e5361f19731d31 [file]
<!DOCTYPE html>
<html lang="en" data-content_root="../../" data-theme="auto">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Distributed query engines &#8212; Apache DataFusion in Python documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
<link rel="stylesheet" type="text/css" href="../../_static/graphviz.css?v=4ae1632d" />
<link rel="stylesheet" type="text/css" href="../../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=4af573bc" />
<!-- So that users can add custom icons -->
<script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../../_static/copybutton.js?v=f281be69"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/distributing-work/query-engines';</script>
<script src="../../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../../_static/favicon.svg"/>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="SQL" href="../sql.html" />
<link rel="prev" title="Shipping expressions to workers" href="expressions.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../../index.html">
<img src="../../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/>
<img src="../../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../index.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../extension-guide/index.html">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../index.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../extension-guide/index.html">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links" aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav">
<ul class="current nav bd-sidenav">
<li class="toctree-l1 current active has-children"><a class="reference internal" href="../index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../basics.html">Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-sources.html">Data Sources</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../dataframe/rendering.html">DataFrame Rendering</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe/execution-metrics.html">Execution Metrics</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/views.html">Registering Views</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/basic-info.html">Basic Operations</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/select-and-filter.html">Column Selections</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/expressions.html">Expressions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../io/arrow.html">Arrow</a></li>
<li class="toctree-l3"><a class="reference internal" href="../io/avro.html">Avro</a></li>
<li class="toctree-l3"><a class="reference internal" href="../io/csv.html">CSV</a></li>
<li class="toctree-l3"><a class="reference internal" href="../io/json.html">JSON</a></li>
<li class="toctree-l3"><a class="reference internal" href="../io/parquet.html">Parquet</a></li>
<li class="toctree-l3"><a class="reference internal" href="../io/table_provider.html">Custom Table Provider</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../configuration.html">Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extensions.html">Using extension libraries</a></li>
<li class="toctree-l2 current active has-children"><a class="reference internal" href="index.html">Distributing work</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="expressions.html">Shipping expressions to workers</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">Distributed query engines</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrade-guides.html">Upgrade Guides</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ai-coding-assistants.html">Using AI Coding Assistants</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../extension-guide/index.html">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/why-ffi.html">Why FFI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/capsule-protocol.html">The capsule protocol</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/table-providers.html">Providers and catalogs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/functions.html">Functions and table functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/codecs.html">Extension codecs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/bundles.html">Extension bundles</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/query-planners.html">Query planners</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/other-components.html">Optimizer rules and configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/sessions.html">Sessions, handles, and lifetimes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../extension-guide/checklist.html">Extension author checklist</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../../contributor-guide/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../contributor-guide/pyo3-guidelines.html">PyO3 binding guidelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../contributor-guide/ffi-internals.html">FFI framing internals</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../autoapi/index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../autoapi/datafusion/index.html">datafusion</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/catalog/index.html">datafusion.catalog</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/context/index.html">datafusion.context</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/extensions/index.html">datafusion.extensions</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
</ul>
</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
</ul>
</details></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/io/index.html">datafusion.io</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/ipc/index.html">datafusion.ipc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/object_store/index.html">datafusion.object_store</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/options/index.html">datafusion.options</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/plan/index.html">datafusion.plan</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/record_batch/index.html">datafusion.record_batch</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/substrait/index.html">datafusion.substrait</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/unparser/index.html">datafusion.unparser</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/user_defined/index.html">datafusion.user_defined</a></li>
</ul>
</details></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li>
<li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li>
</ul>
</details></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../index.html" class="nav-link">User Guide</a></li>
<li class="breadcrumb-item"><a href="index.html" class="nav-link">Distributing work</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Distributed query engines</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<section id="distributed-query-engines">
<span id="id1"></span><h1>Distributed query engines<a class="headerlink" href="#distributed-query-engines" title="Link to this heading">#</a></h1>
<p>A distributed query engine takes the query you already wrote and runs
it across several machines for you. You do not partition anything by
hand and you do not change your queries — the engine installs itself
on your session, rewrites the plan into stages, and executes those
stages on its own workers.</p>
<p>This is the counterpart to <a class="reference internal" href="expressions.html#distributed-expressions"><span class="std std-ref">Shipping expressions to workers</span></a>, where
<em>you</em> decide the partitioning and ship one expression per slice of
data. Choose an engine when you want a single query spread across a
cluster; choose expressions when you already know how to split the
work and only need parallelism.</p>
<section id="how-an-engine-attaches-to-your-session">
<h2>How an engine attaches to your session<a class="headerlink" href="#how-an-engine-attaches-to-your-session" title="Link to this heading">#</a></h2>
<p>An engine library ships an object you hand to
<code class="xref py py-meth docutils literal notranslate"><span class="pre">with_extensions()</span></code>:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">my_engine</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">with_extensions</span><span class="p">(</span><span class="n">my_engine</span><span class="o">.</span><span class="n">Extension</span><span class="p">(</span><span class="s2">&quot;scheduler:50050&quot;</span><span class="p">))</span>
<span class="n">ctx</span><span class="o">.</span><span class="n">register_table</span><span class="p">(</span><span class="s2">&quot;events&quot;</span><span class="p">,</span> <span class="n">my_engine</span><span class="o">.</span><span class="n">TableProvider</span><span class="p">(</span><span class="s2">&quot;s3://bucket/events&quot;</span><span class="p">))</span>
<span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">&quot;SELECT country, count(*) FROM events GROUP BY country&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">sql</span></code> call is unchanged from a single-process program. What
changed is underneath it: installing the engine gave the session a
query planner of the engine’s own, and that planner is what turns
your plan into distributed stages.</p>
<p>Two consequences worth knowing:</p>
<ul class="simple">
<li><p><strong>The engine must be installed before you run the query</strong>, not
before you register tables. Registration order does not matter;
<code class="docutils literal notranslate"><span class="pre">with_extensions</span></code> is a session-level setup step.</p></li>
<li><p><strong>The engine has to be able to reach your tables and functions.</strong>
It ships the plan to its workers, so anything the plan references
has to be reconstructible there. Table providers from the engine’s
own library always are. A Python UDF may or may not be — the engine
library documents what it supports, and
<a class="reference internal" href="expressions.html#distributed-udf-portability"><span class="std std-ref">Portability requirements for inline Python UDFs</span></a> describes the constraints that
apply to Python callables crossing a process boundary in general.</p></li>
</ul>
<p>If you install more than one library, pass them in one
<code class="docutils literal notranslate"><span class="pre">with_extensions</span></code> call so they can see each other. See
<a class="reference internal" href="../extensions.html#user-guide-extensions"><span class="std std-ref">Using extension libraries</span></a> for the details of installing extension
libraries, and <a class="reference internal" href="../../extension-guide/index.html#ffi"><span class="std std-ref">Extension Guide</span></a> if you want to write an engine yourself.</p>
</section>
<section id="available-engines">
<h2>Available engines<a class="headerlink" href="#available-engines" title="Link to this heading">#</a></h2>
<p>Query-level distribution is being built upstream. Neither project
below is usable from datafusion-python yet; both sections will fill
in as the integrations land.</p>
<section id="datafusion-distributed">
<h3>datafusion-distributed<a class="headerlink" href="#datafusion-distributed" title="Link to this heading">#</a></h3>
<p>🚧 <em>Work in progress upstream — not yet usable from datafusion-python.</em></p>
<p><a class="reference external" href="https://github.com/apache/datafusion-distributed">datafusion-distributed</a>
splits a single physical plan into stages and runs each stage on a
different worker node. The driver writes a SQL or DataFrame query
once; the runtime handles partitioning, shuffles, and reassembly.</p>
<p>A datafusion-python integration is in development. In the meantime,
<a class="reference internal" href="expressions.html#distributed-expressions"><span class="std std-ref">Shipping expressions to workers</span></a> covers most use cases that do not
require automatic plan partitioning.</p>
</section>
<section id="apache-ballista">
<h3>Apache Ballista<a class="headerlink" href="#apache-ballista" title="Link to this heading">#</a></h3>
<p>🚧 <em>Work in progress upstream — not yet usable from datafusion-python.</em></p>
<p><a class="reference external" href="https://github.com/apache/datafusion-ballista">Apache Ballista</a>
provides distributed query execution on top of DataFusion with a
scheduler / executor model better suited to long-lived cluster
deployments. A datafusion-python integration is on the roadmap.</p>
</section>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="expressions.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Shipping expressions to workers</p>
</div>
</a>
<a class="right-next"
href="../sql.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">SQL</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#how-an-engine-attaches-to-your-session">How an engine attaches to your session</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#available-engines">Available engines</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion-distributed">datafusion-distributed</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#apache-ballista">Apache Ballista</a></li>
</ul>
</li>
</ul>
</nav></div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<!-- Based on pydata_sphinx_theme/footer.html -->
<footer class="footer mt-5 mt-md-0">
<div class="container">
<div class="footer-item">
<p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p>
<p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</div>
</footer>
</body>
</html>