blob: b9abb8202fd87921f67414656ffcce04a54e6137 [file]
<!DOCTYPE html>
<html lang="en" data-content_root="../" data-theme="auto">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>FFI framing internals &#8212; Apache DataFusion in Python documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
<link rel="stylesheet" type="text/css" href="../_static/graphviz.css?v=4ae1632d" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=4af573bc" />
<!-- So that users can add custom icons -->
<script src="../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=9bcbadda"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=f281be69"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'contributor-guide/ffi-internals';</script>
<script src="../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../_static/favicon.svg"/>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="API Reference" href="../autoapi/index.html" />
<link rel="prev" title="PyO3 binding guidelines" href="pyo3-guidelines.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<img src="../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/>
<img src="../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../user-guide/index.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../extension-guide/index.html">
Extension Guide
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../user-guide/index.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../extension-guide/index.html">
Extension Guide
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links" aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav">
<ul class="current nav bd-sidenav">
<li class="toctree-l1 has-children"><a class="reference internal" href="../user-guide/index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/basics.html">Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/data-sources.html">Data Sources</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/dataframe/rendering.html">DataFrame Rendering</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/dataframe/execution-metrics.html">Execution Metrics</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/views.html">Registering Views</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/basic-info.html">Basic Operations</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/select-and-filter.html">Column Selections</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/expressions.html">Expressions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/functions.html">Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/arrow.html">Arrow</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/avro.html">Avro</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/csv.html">CSV</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/json.html">JSON</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/parquet.html">Parquet</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/table_provider.html">Custom Table Provider</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/configuration.html">Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/extensions.html">Using extension libraries</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/distributing-work/index.html">Distributing work</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/distributing-work/expressions.html">Shipping expressions to workers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/distributing-work/query-engines.html">Distributed query engines</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/upgrade-guides.html">Upgrade Guides</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../extension-guide/index.html">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/why-ffi.html">Why FFI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/capsule-protocol.html">The capsule protocol</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/table-providers.html">Providers and catalogs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/functions.html">Functions and table functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/codecs.html">Extension codecs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/bundles.html">Extension bundles</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/query-planners.html">Query planners</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/other-components.html">Optimizer rules and configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/sessions.html">Sessions, handles, and lifetimes</a></li>
<li class="toctree-l2"><a class="reference internal" href="../extension-guide/checklist.html">Extension author checklist</a></li>
</ul>
</details></li>
<li class="toctree-l1 current active has-children"><a class="reference internal" href="index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="pyo3-guidelines.html">PyO3 binding guidelines</a></li>
<li class="toctree-l2 current active"><a class="current reference internal" href="#">FFI framing internals</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../autoapi/index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2 has-children"><a class="reference internal" href="../autoapi/datafusion/index.html">datafusion</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/catalog/index.html">datafusion.catalog</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/context/index.html">datafusion.context</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/extensions/index.html">datafusion.extensions</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
</ul>
</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
</ul>
</details></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/io/index.html">datafusion.io</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/ipc/index.html">datafusion.ipc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/object_store/index.html">datafusion.object_store</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/options/index.html">datafusion.options</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/plan/index.html">datafusion.plan</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/record_batch/index.html">datafusion.record_batch</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/substrait/index.html">datafusion.substrait</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/unparser/index.html">datafusion.unparser</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/user_defined/index.html">datafusion.user_defined</a></li>
</ul>
</details></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li>
<li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li>
</ul>
</details></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="index.html" class="nav-link">Contributor Guide</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">FFI framing internals</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<section id="ffi-framing-internals">
<span id="ffi-internals"></span><h1>FFI framing internals<a class="headerlink" href="#ffi-framing-internals" title="Link to this heading">#</a></h1>
<p>Read this before changing how datafusion-python frames FFI components. You do
<strong>not</strong> need it to write an extension library — that is the
<a class="reference internal" href="../extension-guide/index.html#extension-guide"><span class="std std-ref">Extension Guide</span></a>.</p>
<p>The invariants here are the reasons the extension-facing rules are shaped the
way they are. Each one has an “obvious” simplification that does not work, and
the point of this page is to record why, so the next person does not spend a
week rediscovering it.</p>
<p>For the wire format itself — how a codec id is stored alongside a payload,
routed back on decode, and which two cases stay unframed — see the module
documentation in <code class="docutils literal notranslate"><span class="pre">crates/core/src/codec.rs</span></code>. It is the authority; this page
does not restate it.</p>
<section id="one-session-one-arc-sessioncontext">
<span id="ffi-internals-one-arc"></span><h2>One session, one <code class="docutils literal notranslate"><span class="pre">Arc&lt;SessionContext&gt;</span></code><a class="headerlink" href="#one-session-one-arc-sessioncontext" title="Link to this heading">#</a></h2>
<p>Every codec handed to a foreign object carries an <code class="docutils literal notranslate"><span class="pre">FFI_TaskContextProvider</span></code>,
and that type holds its provider <strong>weakly</strong>. A registered catalog provider
upgrades the handle on every <code class="docutils literal notranslate"><span class="pre">supports_filters_pushdown</span></code> and every <code class="docutils literal notranslate"><span class="pre">scan</span></code>.
Those handles are bound to one particular <code class="docutils literal notranslate"><span class="pre">Arc&lt;SessionContext&gt;</span></code> allocation, not
to the logical session, so anything that replaces the allocation orphans all of
them and the next query fails with <code class="docutils literal notranslate"><span class="pre">TaskContextProvider</span> <span class="pre">went</span> <span class="pre">out</span> <span class="pre">of</span> <span class="pre">scope</span> <span class="pre">over</span> <span class="pre">FFI</span> <span class="pre">boundary</span></code>.</p>
<p>So a <code class="docutils literal notranslate"><span class="pre">PySessionContext</span></code> keeps the <code class="docutils literal notranslate"><span class="pre">Arc&lt;SessionContext&gt;</span></code> it was created with for
its whole life. Installing a query planner writes the new <code class="docutils literal notranslate"><span class="pre">SessionState</span></code> back
through <code class="docutils literal notranslate"><span class="pre">state_ref()</span></code>, exactly as <code class="docutils literal notranslate"><span class="pre">add_physical_optimizer_rule</span></code> does, rather
than deriving a replacement context. The session id is carried across that
rewrite — <code class="docutils literal notranslate"><span class="pre">SessionStateBuilder</span></code> mints a fresh one otherwise — so <code class="docutils literal notranslate"><span class="pre">session_id()</span></code>
and every <code class="docutils literal notranslate"><span class="pre">TaskContext</span></code> the session hands out keep agreeing.</p>
<p>Repairing the damage instead of avoiding it does not work in general. A context
can rebuild the codecs it holds in its own fields, but a codec already embedded
in a registered <code class="docutils literal notranslate"><span class="pre">FFI_CatalogProvider</span></code> — and in every <code class="docutils literal notranslate"><span class="pre">FFI_SchemaProvider</span></code> and
<code class="docutils literal notranslate"><span class="pre">FFI_TableProvider</span></code> minted from it — is not reachable from Python at all. Nor
can the codec simply retain the session that built it: a codec handed to a
provider is routinely registered straight back into that same session, which
would close the cycle
<code class="docutils literal notranslate"><span class="pre">SessionContext</span> <span class="pre">-&gt;</span> <span class="pre">catalog</span> <span class="pre">-&gt;</span> <span class="pre">FFI</span> <span class="pre">provider</span> <span class="pre">-&gt;</span> <span class="pre">FFI</span> <span class="pre">codec</span> <span class="pre">-&gt;</span> <span class="pre">SessionContext</span></code> and
leak it.</p>
<p><code class="docutils literal notranslate"><span class="pre">SessionContext.enable_url_table</span></code> is the one exception. It clones the
underlying <code class="docutils literal notranslate"><span class="pre">SessionContext</span></code>, so the returned context has an allocation of its
own and must not outlive the receiver. It also forks the session’s state while
keeping its id, so two handles report one <code class="docutils literal notranslate"><span class="pre">session_id()</span></code> with divergent
configuration. That is a bug rather than a design, tracked in
<a class="reference external" href="https://github.com/apache/datafusion-python/issues/1708">apache/datafusion-python#1708</a>;
do not copy the pattern.</p>
</section>
<section id="why-planner-codec-rebinding-is-one-level-deep">
<span id="ffi-internals-rebinding"></span><h2>Why planner codec rebinding is one level deep<a class="headerlink" href="#why-planner-codec-rebinding-is-one-level-deep" title="Link to this heading">#</a></h2>
<p>Installing a codec on a session that already has a foreign planner rebuilds
that planner against the new chain. The rebuild swaps the codecs on the
installed <code class="docutils literal notranslate"><span class="pre">ForeignQueryPlanner</span></code> handle, and only that handle. A planner that
wraps a fallback resolved that fallback when <em>it</em> was installed, and holds the
result inside its own library’s private data — behind a <code class="docutils literal notranslate"><span class="pre">create_physical_plan</span></code>
function pointer, with no Python-side handle. A codec installed afterwards
therefore reaches the outer planner and not the fallback, which keeps whichever
codecs were in force when it was imported.</p>
<p>Neither side can repair that:</p>
<ul class="simple">
<li><p><strong>The host cannot reach it.</strong> <code class="docutils literal notranslate"><span class="pre">FFI_QueryPlanner::new_with_ffi_codecs</span></code> unwraps
exactly one <code class="docutils literal notranslate"><span class="pre">ForeignQueryPlanner</span></code> layer. There is no deeper handle to unwrap
— the same situation as a codec embedded in a registered
<code class="docutils literal notranslate"><span class="pre">FFI_CatalogProvider</span></code>.</p></li>
<li><p><strong>The planner library cannot re-derive it.</strong> <code class="docutils literal notranslate"><span class="pre">FFI_QueryPlanner</span></code> holds its
codecs by value, and <code class="docutils literal notranslate"><span class="pre">Session</span></code> exposes no accessor for the ones the host
currently has, so <code class="docutils literal notranslate"><span class="pre">create_physical_plan</span></code> cannot pick them up from the session
it is handed. The rebuild has to be eager, and an eager rebuild only sees the
top layer.</p></li>
</ul>
<p>A fix has to come from upstream, and is tracked in
<a class="reference external" href="https://github.com/apache/datafusion/issues/24762">apache/datafusion#24762</a>.</p>
<p>The stale codecs stay usable rather than dangling — they hold weak handles to
the one <code class="docutils literal notranslate"><span class="pre">Arc&lt;SessionContext&gt;</span></code> that the previous section keeps alive — so the
effect is a fallback hop serializing with an older codec, not a failure. It is
also invisible to the examples in this repository, which use one fallback in
the same cdylib as its wrapper; <code class="docutils literal notranslate"><span class="pre">datafusion-ffi</span></code> short-circuits a same-library
hop rather than serializing, so no codec runs. A fallback in a <em>different</em>
library would serialize, and would do it with the codecs it was imported with.</p>
<p>The extension-facing consequence — install codecs before a layered planner, and
prefer <code class="docutils literal notranslate"><span class="pre">with_extensions</span></code> — is documented at <a class="reference internal" href="../extension-guide/query-planners.html#planner-codec-rebinding"><span class="std std-ref">Install codecs before a layered planner</span></a>.</p>
</section>
<section id="why-with-extensions-commits-last">
<span id="ffi-internals-commit-order"></span><h2>Why <code class="docutils literal notranslate"><span class="pre">with_extensions</span></code> commits last<a class="headerlink" href="#why-with-extensions-commits-last" title="Link to this heading">#</a></h2>
<p><code class="docutils literal notranslate"><span class="pre">with_extensions</span></code> promises that a bundle which raises leaves the session as it
was, apart from anything a hook writes to the context it is handed
(<a class="reference internal" href="../extension-guide/bundles.html#extension-bundles-transaction"><span class="std std-ref">Failure and rollback</span></a>). Keeping that promise is an ordering constraint on the implementation, not
a property of any one step, because the planner is bound on the shared
<code class="docutils literal notranslate"><span class="pre">SessionState</span></code> rather than on the returned handle.</p>
<p>A call therefore splits into four steps, of which only the last writes:</p>
<ol class="arabic simple">
<li><p><strong>Collect.</strong> Every <code class="docutils literal notranslate"><span class="pre">__datafusion_session_components__</span></code> runs and its codecs
are gathered. Nothing is installed yet, so a hook that raises here has
touched nothing.</p></li>
<li><p><strong>Chains.</strong> The codecs are assembled into the returned handle. Codec chains
live on that handle rather than on the session, so this step writes nothing
to the session even though it can fail on a bad capsule or a duplicate id.</p></li>
<li><p><strong>Resolve.</strong> Every <code class="docutils literal notranslate"><span class="pre">__datafusion_session_planner__</span></code> runs, in argument order,
against the completed chains, and each supplied planner is exported to a
capsule. Every hook that can raise has run by the end of this step.</p></li>
<li><p><strong>Commit.</strong> The accumulated planner is re-imported from its capsule and
bound, in a single <code class="docutils literal notranslate"><span class="pre">SessionState</span></code> rebuild. The bind is skipped entirely when
the call installed nothing, so an empty call does not drag a planner sitting
on another handle’s codecs onto this one’s.</p></li>
</ol>
<p>Only step 4 touches the session, and it is not itself infallible:
<code class="docutils literal notranslate"><span class="pre">_install_extension_planner</span></code> runs <code class="docutils literal notranslate"><span class="pre">ffi_query_planner_from_pycapsule</span></code> before it
calls <code class="docutils literal notranslate"><span class="pre">set_session_query_planner</span></code>, which cannot fail. The property that keeps
the promise is therefore about order, not about any step being incapable of
raising — every fallible operation, including the ones inside the commit,
completes before the first write.</p>
<p>That is the rule for the next field added to <code class="docutils literal notranslate"><span class="pre">SessionExtensionComponents</span></code>, not
only a description of the current code: a new kind of component must do its
fallible work — importing a capsule, resolving a name — before anything is
written, so no failure can leave the session half-updated.</p>
<p>There would be nothing to roll back to if one did. The returned handle shares one
session with the receiver, so the damage is visible from every other handle;
and undoing a registration is not the same as restoring what it displaced,
because deregistering a function that shadowed a built-in removes the built-in
too. The split is cheaper than an undo log that cannot be written correctly.</p>
<p>The extension-facing statement of this is
<a class="reference internal" href="../extension-guide/bundles.html#extension-bundles-transaction"><span class="std std-ref">Failure and rollback</span></a>, which says only that declaring a
component is safe where registering one during the hook is not.</p>
</section>
<section id="two-argument-kinds-for-one-convention">
<h2>Two argument kinds for one convention<a class="headerlink" href="#two-argument-kinds-for-one-convention" title="Link to this heading">#</a></h2>
<p><code class="docutils literal notranslate"><span class="pre">CapsuleGetterArg</span></code> in <code class="docutils literal notranslate"><span class="pre">crates/util/src/lib.rs</span></code> distinguishes three cases: no
argument, the session, and the host’s logical extension codec as a bare
capsule. Provider and catalog registration passes the codec; the extension
codecs, the query planner, and table functions get the session.</p>
<p>The distinction exists so that a <code class="docutils literal notranslate"><span class="pre">TypeError</span></code> from a getter that refused its
argument can be rewritten into an <code class="docutils literal notranslate"><span class="pre">ImportError</span></code> naming what the getter <em>should</em>
accept. Getting that message wrong sends an extension author to fix the wrong
signature, which is why every capsule getter routes through
<code class="docutils literal notranslate"><span class="pre">call_capsule_getter</span></code> rather than calling <code class="docutils literal notranslate"><span class="pre">getattr</span></code> directly. Three importers
previously each had their own copy of that logic and each missed later
corrections to it.</p>
<p>When adding a hook, decide which arm it needs by what its FFI constructor
requires: a task-context provider means it must have the session, since you
cannot get a provider off a capsule. A codec alone means either works, and
passing the codec keeps the host from handing out session handles it does not
need to.</p>
<p>The extension-facing statement of this is <a class="reference internal" href="../extension-guide/capsule-protocol.html#extension-getter-argument"><span class="std std-ref">What your getter receives</span></a>,
which deliberately describes the argument by capability — “something you can
read the host’s logical codec off” — rather than by type.</p>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="pyo3-guidelines.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">PyO3 binding guidelines</p>
</div>
</a>
<a class="right-next"
href="../autoapi/index.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">API Reference</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#one-session-one-arc-sessioncontext">One session, one <code class="docutils literal notranslate"><span class="pre">Arc&lt;SessionContext&gt;</span></code></a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-planner-codec-rebinding-is-one-level-deep">Why planner codec rebinding is one level deep</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-with-extensions-commits-last">Why <code class="docutils literal notranslate"><span class="pre">with_extensions</span></code> commits last</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#two-argument-kinds-for-one-convention">Two argument kinds for one convention</a></li>
</ul>
</nav></div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<!-- Based on pydata_sphinx_theme/footer.html -->
<footer class="footer mt-5 mt-md-0">
<div class="container">
<div class="footer-item">
<p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p>
<p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</div>
</footer>
</body>
</html>