blob: c11dd5492d48a7f174fbc3840a0051e90a4f6834 [file]
<!DOCTYPE html>
<html lang="en" data-content_root="../" data-theme="auto">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Extension Guide &#8212; Apache DataFusion in Python documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
<link rel="stylesheet" type="text/css" href="../_static/graphviz.css?v=4ae1632d" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=4af573bc" />
<!-- So that users can add custom icons -->
<script src="../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=9bcbadda"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=f281be69"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'extension-guide/index';</script>
<script src="../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../_static/favicon.svg"/>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Why FFI" href="why-ffi.html" />
<link rel="prev" title="Using AI Coding Assistants" href="../user-guide/ai-coding-assistants.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<img src="../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/>
<img src="../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../user-guide/index.html">
User Guide
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="#">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../user-guide/index.html">
User Guide
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="#">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links" aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav">
<ul class="current nav bd-sidenav">
<li class="toctree-l1 has-children"><a class="reference internal" href="../user-guide/index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/basics.html">Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/data-sources.html">Data Sources</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/dataframe/rendering.html">DataFrame Rendering</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/dataframe/execution-metrics.html">Execution Metrics</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/views.html">Registering Views</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/basic-info.html">Basic Operations</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/select-and-filter.html">Column Selections</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/expressions.html">Expressions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/functions.html">Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/arrow.html">Arrow</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/avro.html">Avro</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/csv.html">CSV</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/json.html">JSON</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/parquet.html">Parquet</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/table_provider.html">Custom Table Provider</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/configuration.html">Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/extensions.html">Using extension libraries</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/distributing-work/index.html">Distributing work</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/distributing-work/expressions.html">Shipping expressions to workers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/distributing-work/query-engines.html">Distributed query engines</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/upgrade-guides.html">Upgrade Guides</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li>
</ul>
</details></li>
<li class="toctree-l1 current active has-children"><a class="current reference internal" href="#">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="why-ffi.html">Why FFI</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html">The capsule protocol</a></li>
<li class="toctree-l2"><a class="reference internal" href="table-providers.html">Providers and catalogs</a></li>
<li class="toctree-l2"><a class="reference internal" href="functions.html">Functions and table functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="codecs.html">Extension codecs</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html">Extension bundles</a></li>
<li class="toctree-l2"><a class="reference internal" href="query-planners.html">Query planners</a></li>
<li class="toctree-l2"><a class="reference internal" href="other-components.html">Optimizer rules and configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="sessions.html">Sessions, handles, and lifetimes</a></li>
<li class="toctree-l2"><a class="reference internal" href="checklist.html">Extension author checklist</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/pyo3-guidelines.html">PyO3 binding guidelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/ffi-internals.html">FFI framing internals</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../autoapi/index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2 has-children"><a class="reference internal" href="../autoapi/datafusion/index.html">datafusion</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/catalog/index.html">datafusion.catalog</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/context/index.html">datafusion.context</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/extensions/index.html">datafusion.extensions</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
</ul>
</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
</ul>
</details></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/io/index.html">datafusion.io</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/ipc/index.html">datafusion.ipc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/object_store/index.html">datafusion.object_store</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/options/index.html">datafusion.options</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/plan/index.html">datafusion.plan</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/record_batch/index.html">datafusion.record_batch</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/substrait/index.html">datafusion.substrait</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/unparser/index.html">datafusion.unparser</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/user_defined/index.html">datafusion.user_defined</a></li>
</ul>
</details></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li>
<li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li>
</ul>
</details></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Extension Guide</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<section id="extension-guide">
<span id="ffi"></span><span id="id1"></span><h1>Extension Guide<a class="headerlink" href="#extension-guide" title="Link to this heading">#</a></h1>
<p>This section is for people <strong>writing</strong> a library that plugs into
datafusion-python — a package that contributes table providers, functions, a
catalog, extension codecs, or a query planner, usually written in Rust and
exposed through <a class="reference external" href="https://pyo3.rs">PyO3</a>.</p>
<p>Two neighbouring audiences are served elsewhere:</p>
<ul class="simple">
<li><p><strong>Using</strong> an extension library someone else published:
<a class="reference internal" href="../user-guide/extensions.html#user-guide-extensions"><span class="std std-ref">Using extension libraries</span></a>.</p></li>
<li><p><strong>Changing datafusion-python itself</strong>, including the framing that makes this
protocol work: <a class="reference internal" href="../contributor-guide/index.html"><span class="doc">Contributor Guide</span></a>.</p></li>
</ul>
<p>The protocol described here is a public, versioned contract. When a hook’s
signature changes, the change is documented in
<a class="reference internal" href="../user-guide/upgrade-guides.html"><span class="doc">Upgrade Guides</span></a> with a before-and-after, and objects built
against a different DataFusion major version are rejected with a clear error
rather than used as-is.</p>
<section id="start-here">
<h2>Start here<a class="headerlink" href="#start-here" title="Link to this heading">#</a></h2>
<p>If you have not built an extension before, read <a class="reference internal" href="why-ffi.html"><span class="doc">Why FFI</span></a> and
<a class="reference internal" href="capsule-protocol.html"><span class="doc">The capsule protocol</span></a> in order. Together they explain why your library must
not depend on the <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> crate, and the one convention every hook
follows. After that the pages are independent — go to the one matching what
you are contributing.</p>
</section>
<section id="the-three-roles-in-a-query">
<h2>The three roles in a query<a class="headerlink" href="#the-three-roles-in-a-query" title="Link to this heading">#</a></h2>
<p>A single query can involve three independent native libraries, and much of
this section only makes sense once they are distinct in your head:</p>
<ul class="simple">
<li><p><strong>datafusion-python</strong> — the host. Owns the session, and decodes whatever
comes back from the other two.</p></li>
<li><p><strong>A provider library</strong> — owns table providers, catalogs, and functions, plus
the codecs that serialize them.</p></li>
<li><p><strong>A planner library</strong> — owns a query planner and the configuration it needs.</p></li>
</ul>
<p>The worked examples in this repository use two separate crates,
<a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example"><code class="docutils literal notranslate"><span class="pre">datafusion-ffi-example</span></code></a> and <a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-query-planner-example"><code class="docutils literal notranslate"><span class="pre">datafusion-ffi-query-planner-example</span></code></a>, so
each role has a distinct shared-library identity. A real library may play more
than one role; keeping them separate in the examples is what makes the
boundaries observable.</p>
<p>The session owns the codecs used for the exchange and supplies them to the
foreign planner. That is what lets the planner decode provider-owned objects,
and lets datafusion-python decode the physical plan the planner returns.</p>
</section>
<section id="hook-reference">
<h2>Hook reference<a class="headerlink" href="#hook-reference" title="Link to this heading">#</a></h2>
<p>Every integration point is a dunder method named <code class="docutils literal notranslate"><span class="pre">__datafusion_*__</span></code>. The
convention is uniform enough to be worth stating once: your object exposes the
getter, datafusion-python calls it, and it returns a <code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code> wrapping an
FFI-safe struct. See <a class="reference internal" href="capsule-protocol.html"><span class="doc">The capsule protocol</span></a> for what that means and
<a class="reference internal" href="capsule-protocol.html#extension-getter-argument"><span class="std std-ref">What your getter receives</span></a> for the argument every getter in the middle
group receives.</p>
<p>Each hook’s capsule name follows from its own name by
<a class="reference internal" href="capsule-protocol.html#extension-capsule-protocol"><span class="std std-ref">the naming rule</span></a>, so it is not repeated
here.</p>
<div class="pst-scrollable-table-container"><table class="table">
<thead>
<tr class="row-odd"><th class="head"><p>Hook</p></th>
<th class="head"><p>Contributes</p></th>
<th class="head"><p>Argument</p></th>
<th class="head"><p>Documented on</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_table_provider__</span></code></p></td>
<td><p>one table</p></td>
<td><p>codec source</p></td>
<td><p><a class="reference internal" href="table-providers.html"><span class="doc">Providers and catalogs</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_table_provider_factory__</span></code></p></td>
<td><p>tables built by <code class="docutils literal notranslate"><span class="pre">CREATE</span> <span class="pre">EXTERNAL</span> <span class="pre">TABLE</span></code></p></td>
<td><p>codec source</p></td>
<td><p><a class="reference internal" href="table-providers.html"><span class="doc">Providers and catalogs</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_catalog_provider__</span></code></p></td>
<td><p>a named set of schemas</p></td>
<td><p>codec source</p></td>
<td><p><a class="reference internal" href="table-providers.html"><span class="doc">Providers and catalogs</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_catalog_provider_list__</span></code></p></td>
<td><p>the whole catalog namespace</p></td>
<td><p>codec source</p></td>
<td><p><a class="reference internal" href="table-providers.html"><span class="doc">Providers and catalogs</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_schema_provider__</span></code></p></td>
<td><p>a named set of tables</p></td>
<td><p>codec source</p></td>
<td><p><a class="reference internal" href="table-providers.html"><span class="doc">Providers and catalogs</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_table_function__</span></code></p></td>
<td><p>a table-valued function</p></td>
<td><p>session</p></td>
<td><p><a class="reference internal" href="functions.html"><span class="doc">Functions and table functions</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_scalar_udf__</span></code></p></td>
<td><p>a scalar function</p></td>
<td><p>none</p></td>
<td><p><a class="reference internal" href="functions.html"><span class="doc">Functions and table functions</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_aggregate_udf__</span></code></p></td>
<td><p>an aggregate function</p></td>
<td><p>none</p></td>
<td><p><a class="reference internal" href="functions.html"><span class="doc">Functions and table functions</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_window_udf__</span></code></p></td>
<td><p>a window function</p></td>
<td><p>none</p></td>
<td><p><a class="reference internal" href="functions.html"><span class="doc">Functions and table functions</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_logical_extension_codec__</span></code></p></td>
<td><p>a logical codec</p></td>
<td><p>session</p></td>
<td><p><a class="reference internal" href="codecs.html"><span class="doc">Extension codecs</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_physical_extension_codec__</span></code></p></td>
<td><p>a physical codec</p></td>
<td><p>session</p></td>
<td><p><a class="reference internal" href="codecs.html"><span class="doc">Extension codecs</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_codec_id__</span></code></p></td>
<td><p>the wire id a codec’s payloads carry</p></td>
<td><p>—</p></td>
<td><p><a class="reference internal" href="codecs.html#extension-codec-ids"><span class="std std-ref">Codec ids</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_query_planner__</span></code></p></td>
<td><p>a query planner</p></td>
<td><p>session</p></td>
<td><p><a class="reference internal" href="query-planners.html"><span class="doc">Query planners</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_session_components__</span></code></p></td>
<td><p>a bundle’s codecs</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">ctx</span></code></p></td>
<td><p><a class="reference internal" href="bundles.html"><span class="doc">Extension bundles</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_session_planner__</span></code></p></td>
<td><p>a bundle’s planner, wrapping <code class="docutils literal notranslate"><span class="pre">fallback</span></code></p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">ctx</span></code>, <code class="docutils literal notranslate"><span class="pre">fallback</span></code></p></td>
<td><p><a class="reference internal" href="bundles.html"><span class="doc">Extension bundles</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_physical_optimizer_rule__</span></code></p></td>
<td><p>a physical optimizer rule</p></td>
<td><p>none</p></td>
<td><p><a class="reference internal" href="other-components.html#extension-other-hooks"><span class="std std-ref">Optimizer rules and configuration</span></a></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_extension_options__</span></code></p></td>
<td><p>typed entries in the session config</p></td>
<td><p>none</p></td>
<td><p><a class="reference internal" href="other-components.html#extension-other-hooks"><span class="std std-ref">Optimizer rules and configuration</span></a></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">__datafusion_task_context_provider__</span></code></p></td>
<td><p>the host’s task context</p></td>
<td><p>none</p></td>
<td><p><a class="reference internal" href="capsule-protocol.html#extension-task-context-provider"><span class="std std-ref">__datafusion_task_context_provider__</span></a></p></td>
</tr>
</tbody>
</table>
</div>
<p>Three rows are not like the others. <code class="docutils literal notranslate"><span class="pre">__datafusion_task_context_provider__</span></code> is
implemented by the <strong>host</strong>, not by your library — you read it off the session
you are handed. <code class="docutils literal notranslate"><span class="pre">__datafusion_codec_id__</span></code> is a plain string attribute rather
than a method returning a capsule. And the two <code class="docutils literal notranslate"><span class="pre">session_</span></code> hooks are dispatched
from Python and return objects rather than capsules.</p>
<p>“codec source” in the argument column means the value is something you can
read the host’s logical extension codec off, which is not always a session.
<a class="reference internal" href="capsule-protocol.html#extension-getter-argument"><span class="std std-ref">What your getter receives</span></a> explains why, and what to do with it.</p>
<p><code class="docutils literal notranslate"><span class="pre">python/tests/test_docstrings.py</span></code> compares this table against the hook names
the package actually dispatches, so a hook added or renamed without touching
this page fails the suite.</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="why-ffi.html">Why FFI</a><ul>
<li class="toctree-l2"><a class="reference internal" href="why-ffi.html#the-primary-issue">The primary issue</a></li>
<li class="toctree-l2"><a class="reference internal" href="why-ffi.html#the-ffi-approach">The FFI approach</a></li>
<li class="toctree-l2"><a class="reference internal" href="why-ffi.html#inspiration-from-arrow">Inspiration from Arrow</a></li>
<li class="toctree-l2"><a class="reference internal" href="why-ffi.html#if-ffi-does-not-yet-cover-what-you-need">If FFI does not yet cover what you need</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="capsule-protocol.html">The capsule protocol</a><ul>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html#ffi-safe-types">FFI-safe types</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html#ffi-on-the-provider-foreign-on-the-receiver"><code class="docutils literal notranslate"><span class="pre">FFI_</span></code> on the provider, <code class="docutils literal notranslate"><span class="pre">Foreign</span></code> on the receiver</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html#wrapping-it-in-a-capsule">Wrapping it in a capsule</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html#the-naming-rule">The naming rule</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html#version-checking">Version checking</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html#what-your-getter-receives">What your getter receives</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html#datafusion-task-context-provider"><code class="docutils literal notranslate"><span class="pre">__datafusion_task_context_provider__</span></code></a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="table-providers.html">Providers and catalogs</a><ul>
<li class="toctree-l2"><a class="reference internal" href="table-providers.html#a-table-provider">A table provider</a></li>
<li class="toctree-l2"><a class="reference internal" href="table-providers.html#the-catalog-family">The catalog family</a></li>
<li class="toctree-l2"><a class="reference internal" href="table-providers.html#a-table-provider-factory">A table provider factory</a></li>
<li class="toctree-l2"><a class="reference internal" href="table-providers.html#serializing-what-you-expose">Serializing what you expose</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="functions.html">Functions and table functions</a><ul>
<li class="toctree-l2"><a class="reference internal" href="functions.html#the-three-scalar-shaped-hooks">The three scalar-shaped hooks</a></li>
<li class="toctree-l2"><a class="reference internal" href="functions.html#table-functions">Table functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="functions.html#serializing-functions">Serializing functions</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="codecs.html">Extension codecs</a><ul>
<li class="toctree-l2"><a class="reference internal" href="codecs.html#codecs-compose">Codecs compose</a></li>
<li class="toctree-l2"><a class="reference internal" href="codecs.html#encode-metadata-not-a-handle-to-a-live-object">Encode metadata, not a handle to a live object</a></li>
<li class="toctree-l2"><a class="reference internal" href="codecs.html#codec-ids">Codec ids</a></li>
<li class="toctree-l2"><a class="reference internal" href="codecs.html#functions-whose-name-is-the-whole-encoding">Functions whose name is the whole encoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="codecs.html#when-codec-order-matters">When codec order matters</a></li>
<li class="toctree-l2"><a class="reference internal" href="codecs.html#a-codec-decodes-against-the-session-running-the-query">A codec decodes against the session running the query</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="bundles.html">Extension bundles</a><ul>
<li class="toctree-l2"><a class="reference internal" href="bundles.html#why-not-have-callers-install-the-pieces">Why not have callers install the pieces</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html#a-bundle-you-can-run">A bundle you can run</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html#two-phases-because-codecs-and-planners-compose-differently">Two phases, because codecs and planners compose differently</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html#returning-a-planner-or-not">Returning a planner, or not</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html#when-the-two-orders-conflict">When the two orders conflict</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html#codecs-are-objects-not-capsules">Codecs are objects, not capsules</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html#failure-and-rollback">Failure and rollback</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="query-planners.html">Query planners</a><ul>
<li class="toctree-l2"><a class="reference internal" href="query-planners.html#one-planner-per-session">One planner per session</a></li>
<li class="toctree-l2"><a class="reference internal" href="query-planners.html#install-codecs-before-a-layered-planner">Install codecs before a layered planner</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="other-components.html">Optimizer rules and configuration</a><ul>
<li class="toctree-l2"><a class="reference internal" href="other-components.html#a-physical-optimizer-rule">A physical optimizer rule</a></li>
<li class="toctree-l2"><a class="reference internal" href="other-components.html#typed-configuration">Typed configuration</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="sessions.html">Sessions, handles, and lifetimes</a><ul>
<li class="toctree-l2"><a class="reference internal" href="sessions.html#the-rule-that-surprises-people">The rule that surprises people</a></li>
<li class="toctree-l2"><a class="reference internal" href="sessions.html#what-a-derived-context-shares">What a derived context shares</a></li>
<li class="toctree-l2"><a class="reference internal" href="sessions.html#keep-a-context-alive">Keep a context alive</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="checklist.html">Extension author checklist</a><ul>
<li class="toctree-l2"><a class="reference internal" href="checklist.html#protocol">Protocol</a></li>
<li class="toctree-l2"><a class="reference internal" href="checklist.html#codecs">Codecs</a></li>
<li class="toctree-l2"><a class="reference internal" href="checklist.html#bundles-and-planners">Bundles and planners</a></li>
<li class="toctree-l2"><a class="reference internal" href="checklist.html#packaging-and-documentation">Packaging and documentation</a></li>
</ul>
</li>
</ul>
</div>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="../user-guide/ai-coding-assistants.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Using AI Coding Assistants</p>
</div>
</a>
<a class="right-next"
href="why-ffi.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">Why FFI</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#start-here">Start here</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-three-roles-in-a-query">The three roles in a query</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#hook-reference">Hook reference</a></li>
</ul>
</nav></div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<!-- Based on pydata_sphinx_theme/footer.html -->
<footer class="footer mt-5 mt-md-0">
<div class="container">
<div class="footer-item">
<p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p>
<p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</div>
</footer>
</body>
</html>