blob: 5668d8ff27b0721f38ef2c840511e9d9f3667317 [file]
<!DOCTYPE html>
<html lang="en" data-content_root="../" data-theme="auto">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Configuration &#8212; Apache DataFusion in Python documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
<link rel="stylesheet" type="text/css" href="../_static/graphviz.css?v=4ae1632d" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=4af573bc" />
<!-- So that users can add custom icons -->
<script src="../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=9bcbadda"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=f281be69"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/configuration';</script>
<script src="../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../_static/favicon.svg"/>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Using extension libraries" href="extensions.html" />
<link rel="prev" title="Custom Table Provider" href="io/table_provider.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<img src="../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/>
<img src="../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item current active">
<a class="nav-link nav-internal" href="index.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../extension-guide/index.html">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item current active">
<a class="nav-link nav-internal" href="index.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../extension-guide/index.html">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links" aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav">
<ul class="current nav bd-sidenav">
<li class="toctree-l1 current active has-children"><a class="reference internal" href="index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="basics.html">Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="data-sources.html">Data Sources</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="dataframe/rendering.html">DataFrame Rendering</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataframe/execution-metrics.html">Execution Metrics</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="common-operations/views.html">Registering Views</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/basic-info.html">Basic Operations</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/select-and-filter.html">Column Selections</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/expressions.html">Expressions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="io/arrow.html">Arrow</a></li>
<li class="toctree-l3"><a class="reference internal" href="io/avro.html">Avro</a></li>
<li class="toctree-l3"><a class="reference internal" href="io/csv.html">CSV</a></li>
<li class="toctree-l3"><a class="reference internal" href="io/json.html">JSON</a></li>
<li class="toctree-l3"><a class="reference internal" href="io/parquet.html">Parquet</a></li>
<li class="toctree-l3"><a class="reference internal" href="io/table_provider.html">Custom Table Provider</a></li>
</ul>
</details></li>
<li class="toctree-l2 current active"><a class="current reference internal" href="#">Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="extensions.html">Using extension libraries</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="distributing-work/index.html">Distributing work</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="distributing-work/expressions.html">Shipping expressions to workers</a></li>
<li class="toctree-l3"><a class="reference internal" href="distributing-work/query-engines.html">Distributed query engines</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="upgrade-guides.html">Upgrade Guides</a></li>
<li class="toctree-l2"><a class="reference internal" href="ai-coding-assistants.html">Using AI Coding Assistants</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../extension-guide/index.html">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/why-ffi.html">Why FFI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/capsule-protocol.html">The capsule protocol</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/table-providers.html">Providers and catalogs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/functions.html">Functions and table functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/codecs.html">Extension codecs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/bundles.html">Extension bundles</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/query-planners.html">Query planners</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/other-components.html">Optimizer rules and configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/sessions.html">Sessions, handles, and lifetimes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/checklist.html">Extension author checklist</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/pyo3-guidelines.html">PyO3 binding guidelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/ffi-internals.html">FFI framing internals</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../autoapi/index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2 has-children"><a class="reference internal" href="../autoapi/datafusion/index.html">datafusion</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/catalog/index.html">datafusion.catalog</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/context/index.html">datafusion.context</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/extensions/index.html">datafusion.extensions</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
</ul>
</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
</ul>
</details></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/io/index.html">datafusion.io</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/ipc/index.html">datafusion.ipc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/object_store/index.html">datafusion.object_store</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/options/index.html">datafusion.options</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/plan/index.html">datafusion.plan</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/record_batch/index.html">datafusion.record_batch</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/substrait/index.html">datafusion.substrait</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/unparser/index.html">datafusion.unparser</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/user_defined/index.html">datafusion.user_defined</a></li>
</ul>
</details></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li>
<li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li>
</ul>
</details></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="index.html" class="nav-link">User Guide</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Configuration</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<section id="configuration">
<span id="id1"></span><h1>Configuration<a class="headerlink" href="#configuration" title="Link to this heading">#</a></h1>
<p>Let’s look at how we can configure DataFusion. When creating a <a class="reference internal" href="../autoapi/datafusion/context/index.html#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>, you can pass in
a <a class="reference internal" href="../autoapi/datafusion/context/index.html#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionConfig</span></code></a> and <a class="reference internal" href="../autoapi/datafusion/context/index.html#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-class docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a> object. These two cover a wide range of options.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">RuntimeEnvBuilder</span><span class="p">,</span> <span class="n">SessionConfig</span><span class="p">,</span> <span class="n">SessionContext</span>
<span class="c1"># create a session context with default settings</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="n">ctx</span><span class="p">)</span>
<span class="c1"># create a session context with explicit runtime and config settings</span>
<span class="n">runtime</span> <span class="o">=</span> <span class="n">RuntimeEnvBuilder</span><span class="p">()</span><span class="o">.</span><span class="n">with_disk_manager_os</span><span class="p">()</span><span class="o">.</span><span class="n">with_fair_spill_pool</span><span class="p">(</span><span class="mi">10000000</span><span class="p">)</span>
<span class="n">config</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">SessionConfig</span><span class="p">()</span>
<span class="o">.</span><span class="n">with_create_default_catalog_and_schema</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_default_catalog_and_schema</span><span class="p">(</span><span class="s2">&quot;foo&quot;</span><span class="p">,</span> <span class="s2">&quot;bar&quot;</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_target_partitions</span><span class="p">(</span><span class="mi">8</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_information_schema</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_repartition_joins</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_repartition_aggregations</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_repartition_windows</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_parquet_pruning</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>
<span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">&quot;datafusion.execution.parquet.pushdown_filters&quot;</span><span class="p">,</span> <span class="s2">&quot;true&quot;</span><span class="p">)</span>
<span class="p">)</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">config</span><span class="p">,</span> <span class="n">runtime</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">ctx</span><span class="p">)</span>
</pre></div>
</div>
<section id="setting-options-by-key">
<h2>Setting options by key<a class="headerlink" href="#setting-options-by-key" title="Link to this heading">#</a></h2>
<p>The <code class="docutils literal notranslate"><span class="pre">with_*</span></code> methods cover the common options, but any option DataFusion declares can be
set by its fully qualified key with <a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.SessionConfig.set" title="datafusion.SessionConfig.set"><code class="xref py py-meth docutils literal notranslate"><span class="pre">set()</span></code></a>. The value is
always a string, and is parsed according to the type the option declares, so an unknown key
or an unparsable value raises rather than being silently ignored:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="n">SessionConfig</span><span class="p">()</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">&quot;datafusion.execution.batch_size&quot;</span><span class="p">,</span> <span class="s2">&quot;1024&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>A whole dictionary of options can be applied at once by passing it to the
<a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.SessionConfig" title="datafusion.SessionConfig"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionConfig</span></code></a> constructor, which is the shape a replayed set of
settings usually arrives in:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="n">SessionConfig</span><span class="p">({</span><span class="s2">&quot;datafusion.execution.batch_size&quot;</span><span class="p">:</span> <span class="s2">&quot;1024&quot;</span><span class="p">})</span>
</pre></div>
</div>
<p>Both routes reject the same keys, so which one you use does not change what is accepted. The
constructor applies its entries in an unspecified order, so a dictionary with more than one
bad key does not report a predictable one first.</p>
<p>One trap is worth knowing about if you read settings back out of a session and replay them
somewhere else, such as onto a worker process or into a test fixture. With
<code class="docutils literal notranslate"><span class="pre">with_information_schema(True)</span></code>, the <code class="docutils literal notranslate"><span class="pre">information_schema.df_settings</span></code> table lists the
<code class="docutils literal notranslate"><span class="pre">datafusion.runtime.*</span></code> keys alongside the rest, but those come from the runtime environment
rather than from <code class="docutils literal notranslate"><span class="pre">ConfigOptions</span></code> and cannot be set this way. Feeding that table’s rows back
in verbatim will fail on the first such row, whichever route you use. Configure the runtime
through <code class="docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code> instead, and skip the <code class="docutils literal notranslate"><span class="pre">datafusion.runtime.</span></code> prefix when
replaying.</p>
</section>
<section id="maximizing-cpu-usage">
<h2>Maximizing CPU Usage<a class="headerlink" href="#maximizing-cpu-usage" title="Link to this heading">#</a></h2>
<p>DataFusion uses partitions to parallelize work. For small queries the
default configuration (number of CPU cores) is often sufficient, but to
fully utilize available hardware you can tune how many partitions are
created and when DataFusion will repartition data automatically.</p>
<p>Configure a <code class="docutils literal notranslate"><span class="pre">SessionContext</span></code> with a higher partition count:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionConfig</span><span class="p">,</span> <span class="n">SessionContext</span>
<span class="c1"># allow up to 16 concurrent partitions</span>
<span class="n">config</span> <span class="o">=</span> <span class="n">SessionConfig</span><span class="p">()</span><span class="o">.</span><span class="n">with_target_partitions</span><span class="p">(</span><span class="mi">16</span><span class="p">)</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
</pre></div>
</div>
<p>Automatic repartitioning for joins, aggregations, window functions and
other operations can be enabled to increase parallelism:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">SessionConfig</span><span class="p">()</span>
<span class="o">.</span><span class="n">with_target_partitions</span><span class="p">(</span><span class="mi">16</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_repartition_joins</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_repartition_aggregations</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
<span class="o">.</span><span class="n">with_repartition_windows</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
<span class="p">)</span>
</pre></div>
</div>
<p>Manual repartitioning is available on DataFrames when you need precise
control:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">read_parquet</span><span class="p">(</span><span class="s2">&quot;data.parquet&quot;</span><span class="p">)</span>
<span class="c1"># Evenly divide into 16 partitions</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">repartition</span><span class="p">(</span><span class="mi">16</span><span class="p">)</span>
<span class="c1"># Or partition by the hash of a column</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">repartition_by_hash</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;a&quot;</span><span class="p">),</span> <span class="n">num</span><span class="o">=</span><span class="mi">16</span><span class="p">)</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
</pre></div>
</div>
<section id="checking-what-the-plan-actually-does">
<span id="checking-partitioning"></span><h3>Checking what the plan actually does<a class="headerlink" href="#checking-what-the-plan-actually-does" title="Link to this heading">#</a></h3>
<p><code class="docutils literal notranslate"><span class="pre">repartition</span></code> and <code class="docutils literal notranslate"><span class="pre">repartition_by_hash</span></code> are requests, not instructions. The optimizer is
free to drop a repartition nothing downstream needs, to collapse partitions again for an
operator that requires a single stream, or to substitute a repartition of its own sized by
<code class="docutils literal notranslate"><span class="pre">target_partitions</span></code>. So the number you passed is not necessarily the number you get.</p>
<p><a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.ExecutionPlan.output_partitioning" title="datafusion.ExecutionPlan.output_partitioning"><code class="xref py py-attr docutils literal notranslate"><span class="pre">output_partitioning</span></code></a> reports what the built plan does,
as opposed to what was asked of it:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionConfig</span><span class="p">,</span> <span class="n">SessionContext</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">functions</span> <span class="k">as</span> <span class="n">f</span>
<span class="n">config</span> <span class="o">=</span> <span class="n">SessionConfig</span><span class="p">()</span><span class="o">.</span><span class="n">with_target_partitions</span><span class="p">(</span><span class="mi">16</span><span class="p">)</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">read_parquet</span><span class="p">(</span><span class="s2">&quot;data.parquet&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">repartition_by_hash</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;a&quot;</span><span class="p">),</span> <span class="n">num</span><span class="o">=</span><span class="mi">8</span><span class="p">)</span>
<span class="n">plan</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">([</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;a&quot;</span><span class="p">)],</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">&quot;b&quot;</span><span class="p">))])</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span>
<span class="n">partitioning</span> <span class="o">=</span> <span class="n">plan</span><span class="o">.</span><span class="n">output_partitioning</span>
<span class="nb">print</span><span class="p">(</span><span class="n">partitioning</span><span class="o">.</span><span class="n">scheme</span><span class="p">)</span> <span class="c1"># &#39;Hash&#39;</span>
<span class="nb">print</span><span class="p">(</span><span class="n">partitioning</span><span class="o">.</span><span class="n">partition_count</span><span class="p">)</span> <span class="c1"># 16 -- target_partitions, not the 8 requested</span>
<span class="nb">print</span><span class="p">(</span><span class="n">partitioning</span><span class="o">.</span><span class="n">hash_expressions</span><span class="p">)</span> <span class="c1"># [&#39;a@0&#39;]</span>
</pre></div>
</div>
<p>The request for eight partitions did not survive: the optimizer inserted its own hash
repartition at <code class="docutils literal notranslate"><span class="pre">target_partitions</span></code> instead. Had the aggregation been left off, the
repartition would have been removed altogether and the plan would report
<code class="docutils literal notranslate"><span class="pre">UnknownPartitioning</span></code> over the source’s own partition count.</p>
<p><code class="docutils literal notranslate"><span class="pre">UnknownPartitioning</span></code> means the plan knows how many partitions it has but nothing about how
rows are distributed across them, which is the ordinary case for a file scan.
<a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.ExecutionPlan.partition_count" title="datafusion.ExecutionPlan.partition_count"><code class="xref py py-attr docutils literal notranslate"><span class="pre">partition_count</span></code></a> gives the same count on its own when
the scheme does not matter.</p>
<p>Four schemes exist, but only three of them can come out of a plan you built here.
<code class="docutils literal notranslate"><span class="pre">UnknownPartitioning</span></code> comes from a source, and <code class="docutils literal notranslate"><span class="pre">RoundRobinBatch</span></code> and <code class="docutils literal notranslate"><span class="pre">Hash</span></code> from a
repartition — either one you asked for or one the optimizer inserted. <code class="docutils literal notranslate"><span class="pre">Range</span></code>, which spreads
an ordered key space across partitions at chosen split points, has no request form in this
package: <code class="docutils literal notranslate"><span class="pre">repartition</span></code> asks for round-robin, <code class="docutils literal notranslate"><span class="pre">repartition_by_hash</span></code> asks for hash, and SQL
has no range-repartition syntax.</p>
<p>It is still worth handling, because a plan does not have to have been built here. Both
<code class="docutils literal notranslate"><span class="pre">datafusion-proto</span></code> and <code class="docutils literal notranslate"><span class="pre">datafusion-ffi</span></code> carry range partitioning faithfully, so
<a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.ExecutionPlan.from_bytes" title="datafusion.ExecutionPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_bytes()</span></code></a> can return a plan reporting it, as can an
extension library whose query planner builds one — see <a class="reference internal" href="../extension-guide/query-planners.html#extension-planners"><span class="std std-ref">Query planners</span></a>. Such a
plan executes normally; only the split points are invisible, since
<a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.PhysicalPartitioning.hash_expressions" title="datafusion.PhysicalPartitioning.hash_expressions"><code class="xref py py-attr docutils literal notranslate"><span class="pre">hash_expressions</span></code></a> returns <code class="docutils literal notranslate"><span class="pre">None</span></code> for every scheme
but <code class="docutils literal notranslate"><span class="pre">Hash</span></code>. Read <code class="xref py py-func docutils literal notranslate"><span class="pre">repr()</span></code> of the partitioning to see them.</p>
</section>
<section id="benchmark-example">
<h3>Benchmark Example<a class="headerlink" href="#benchmark-example" title="Link to this heading">#</a></h3>
<p>The repository includes a benchmark script that demonstrates how to maximize CPU usage
with DataFusion. The <code class="code docutils literal notranslate"><span class="pre">benchmarks/max_cpu_usage.py</span></code> script shows a practical example
of configuring DataFusion for optimal parallelism.</p>
<p>You can run the benchmark script to see the impact of different configuration settings:</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Run with default settings (uses all CPU cores)</span>
python<span class="w"> </span>benchmarks/max_cpu_usage.py
<span class="c1"># Run with specific number of rows and partitions</span>
python<span class="w"> </span>benchmarks/max_cpu_usage.py<span class="w"> </span>--rows<span class="w"> </span><span class="m">5000000</span><span class="w"> </span>--partitions<span class="w"> </span><span class="m">16</span>
<span class="c1"># See all available options</span>
python<span class="w"> </span>benchmarks/max_cpu_usage.py<span class="w"> </span>--help
</pre></div>
</div>
<p>Here’s an example showing the performance difference between single and multiple partitions:</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Single partition - slower processing</span>
$<span class="w"> </span>python<span class="w"> </span>benchmarks/max_cpu_usage.py<span class="w"> </span>--rows<span class="o">=</span><span class="m">10000000</span><span class="w"> </span>--partitions<span class="w"> </span><span class="m">1</span>
Processed<span class="w"> </span><span class="m">10000000</span><span class="w"> </span>rows<span class="w"> </span>using<span class="w"> </span><span class="m">1</span><span class="w"> </span>partitions<span class="w"> </span><span class="k">in</span><span class="w"> </span><span class="m">0</span>.107s
<span class="c1"># Multiple partitions - faster processing</span>
$<span class="w"> </span>python<span class="w"> </span>benchmarks/max_cpu_usage.py<span class="w"> </span>--rows<span class="o">=</span><span class="m">10000000</span><span class="w"> </span>--partitions<span class="w"> </span><span class="m">10</span>
Processed<span class="w"> </span><span class="m">10000000</span><span class="w"> </span>rows<span class="w"> </span>using<span class="w"> </span><span class="m">10</span><span class="w"> </span>partitions<span class="w"> </span><span class="k">in</span><span class="w"> </span><span class="m">0</span>.038s
</pre></div>
</div>
<p>This example demonstrates nearly 3x performance improvement (0.107s vs 0.038s) when using
10 partitions instead of 1, showcasing how proper partitioning can significantly improve
CPU utilization and query performance.</p>
<p>The script demonstrates several key optimization techniques:</p>
<ol class="arabic simple">
<li><p><strong>Higher target partition count</strong>: Uses <code class="code docutils literal notranslate"><span class="pre">with_target_partitions()</span></code> to set the number of concurrent partitions</p></li>
<li><p><strong>Automatic repartitioning</strong>: Enables repartitioning for joins, aggregations, and window functions</p></li>
<li><p><strong>Manual repartitioning</strong>: Uses <code class="code docutils literal notranslate"><span class="pre">repartition()</span></code> to ensure all partitions are utilized</p></li>
<li><p><strong>CPU-intensive operations</strong>: Performs aggregations that can benefit from parallelization</p></li>
</ol>
<p>The benchmark creates synthetic data and measures the time taken to perform a sum aggregation
across the specified number of partitions. This helps you understand how partition configuration
affects performance on your specific hardware.</p>
<section id="important-considerations">
<h4>Important Considerations<a class="headerlink" href="#important-considerations" title="Link to this heading">#</a></h4>
<p>The provided benchmark script demonstrates partitioning concepts using synthetic in-memory data
and simple aggregation operations. While useful for understanding basic configuration principles,
actual performance in production environments may vary significantly based on numerous factors:</p>
<p><strong>Data Sources and I/O Characteristics:</strong></p>
<ul class="simple">
<li><p><strong>Table providers</strong>: Performance differs greatly between Parquet files, CSV files, databases, and cloud storage</p></li>
<li><p><strong>Storage type</strong>: Local SSD, network-attached storage, and cloud storage have vastly different characteristics</p></li>
<li><p><strong>Network latency</strong>: Remote data sources introduce additional latency considerations</p></li>
<li><p><strong>File sizes and distribution</strong>: Large files may benefit differently from partitioning than many small files</p></li>
</ul>
<p><strong>Query and Workload Characteristics:</strong></p>
<ul class="simple">
<li><p><strong>Operation complexity</strong>: Simple aggregations versus complex joins, window functions, or nested queries</p></li>
<li><p><strong>Data distribution</strong>: Skewed data may not partition evenly, affecting parallel efficiency</p></li>
<li><p><strong>Memory usage</strong>: Large datasets may require different memory management strategies</p></li>
<li><p><strong>Concurrent workloads</strong>: Multiple queries running simultaneously affect resource allocation</p></li>
</ul>
<p><strong>Hardware and Environment Factors:</strong></p>
<ul class="simple">
<li><p><strong>CPU architecture</strong>: Different processors have varying parallel processing capabilities</p></li>
<li><p><strong>Available memory</strong>: Limited RAM may require different optimization strategies</p></li>
<li><p><strong>System load</strong>: Other applications competing for resources affect DataFusion performance</p></li>
</ul>
<p><strong>Recommendations for Production Use:</strong></p>
<p>To optimize DataFusion for your specific use case, it is strongly recommended to:</p>
<ol class="arabic simple">
<li><p><strong>Create custom benchmarks</strong> using your actual data sources, formats, and query patterns</p></li>
<li><p><strong>Test with representative data volumes</strong> that match your production workloads</p></li>
<li><p><strong>Measure end-to-end performance</strong> including data loading, processing, and result handling</p></li>
<li><p><strong>Evaluate different configuration combinations</strong> for your specific hardware and workload</p></li>
<li><p><strong>Monitor resource utilization</strong> (CPU, memory, I/O) to identify bottlenecks in your environment</p></li>
</ol>
<p>This approach will provide more accurate insights into how DataFusion configuration options
will impact your particular applications and infrastructure.</p>
<p>For more information about available <a class="reference internal" href="../autoapi/datafusion/context/index.html#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionConfig</span></code></a> options, see the <a class="reference external" href="https://arrow.apache.org/datafusion/user-guide/configs.html">rust DataFusion Configuration guide</a>,
and about <code class="code docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code> options in the rust <a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/execution/runtime_env/struct.RuntimeEnvBuilder.html">online API documentation</a>.</p>
</section>
</section>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="io/table_provider.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Custom Table Provider</p>
</div>
</a>
<a class="right-next"
href="extensions.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">Using extension libraries</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#setting-options-by-key">Setting options by key</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#maximizing-cpu-usage">Maximizing CPU Usage</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#checking-what-the-plan-actually-does">Checking what the plan actually does</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#benchmark-example">Benchmark Example</a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#important-considerations">Important Considerations</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</nav></div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<!-- Based on pydata_sphinx_theme/footer.html -->
<footer class="footer mt-5 mt-md-0">
<div class="container">
<div class="footer-item">
<p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p>
<p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</div>
</footer>
</body>
</html>