blob: 66eda44c6963b9afeb5e19187b6a62fbaf985624 [file]
<!DOCTYPE html>
<html lang="en" data-content_root="../" data-theme="auto">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Extension codecs &#8212; Apache DataFusion in Python documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
<link rel="stylesheet" type="text/css" href="../_static/graphviz.css?v=4ae1632d" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=4af573bc" />
<!-- So that users can add custom icons -->
<script src="../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=9bcbadda"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=f281be69"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'extension-guide/codecs';</script>
<script src="../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../_static/favicon.svg"/>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Extension bundles" href="bundles.html" />
<link rel="prev" title="Functions and table functions" href="functions.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<img src="../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/>
<img src="../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../user-guide/index.html">
User Guide
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="index.html">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../user-guide/index.html">
User Guide
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="index.html">
Extension Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../contributor-guide/index.html">
Contributor Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../autoapi/index.html">
API Reference
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../links.html">
Links
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
<span class="sr-only">Rust API docs (docs.rs)</span></a>
</li>
</ul></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links" aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav">
<ul class="current nav bd-sidenav">
<li class="toctree-l1 has-children"><a class="reference internal" href="../user-guide/index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/basics.html">Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/data-sources.html">Data Sources</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/dataframe/rendering.html">DataFrame Rendering</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/dataframe/execution-metrics.html">Execution Metrics</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/views.html">Registering Views</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/basic-info.html">Basic Operations</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/select-and-filter.html">Column Selections</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/expressions.html">Expressions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/functions.html">Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
</ul>
</details></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/arrow.html">Arrow</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/avro.html">Avro</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/csv.html">CSV</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/json.html">JSON</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/parquet.html">Parquet</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/io/table_provider.html">Custom Table Provider</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/configuration.html">Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/extensions.html">Using extension libraries</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../user-guide/distributing-work/index.html">Distributing work</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/distributing-work/expressions.html">Shipping expressions to workers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/distributing-work/query-engines.html">Distributed query engines</a></li>
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/upgrade-guides.html">Upgrade Guides</a></li>
<li class="toctree-l2"><a class="reference internal" href="../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li>
</ul>
</details></li>
<li class="toctree-l1 current active has-children"><a class="reference internal" href="index.html">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="why-ffi.html">Why FFI</a></li>
<li class="toctree-l2"><a class="reference internal" href="capsule-protocol.html">The capsule protocol</a></li>
<li class="toctree-l2"><a class="reference internal" href="table-providers.html">Providers and catalogs</a></li>
<li class="toctree-l2"><a class="reference internal" href="functions.html">Functions and table functions</a></li>
<li class="toctree-l2 current active"><a class="current reference internal" href="#">Extension codecs</a></li>
<li class="toctree-l2"><a class="reference internal" href="bundles.html">Extension bundles</a></li>
<li class="toctree-l2"><a class="reference internal" href="query-planners.html">Query planners</a></li>
<li class="toctree-l2"><a class="reference internal" href="other-components.html">Optimizer rules and configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="sessions.html">Sessions, handles, and lifetimes</a></li>
<li class="toctree-l2"><a class="reference internal" href="checklist.html">Extension author checklist</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/pyo3-guidelines.html">PyO3 binding guidelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../contributor-guide/ffi-internals.html">FFI framing internals</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../autoapi/index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2 has-children"><a class="reference internal" href="../autoapi/datafusion/index.html">datafusion</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/catalog/index.html">datafusion.catalog</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/context/index.html">datafusion.context</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/extensions/index.html">datafusion.extensions</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
</ul>
</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
</ul>
</details></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/io/index.html">datafusion.io</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/ipc/index.html">datafusion.ipc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/object_store/index.html">datafusion.object_store</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/options/index.html">datafusion.options</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/plan/index.html">datafusion.plan</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/record_batch/index.html">datafusion.record_batch</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/substrait/index.html">datafusion.substrait</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/unparser/index.html">datafusion.unparser</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/user_defined/index.html">datafusion.user_defined</a></li>
</ul>
</details></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li>
<li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li>
<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li>
</ul>
</details></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="index.html" class="nav-link">Extension Guide</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Extension codecs</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<section id="extension-codecs">
<span id="id1"></span><h1>Extension codecs<a class="headerlink" href="#extension-codecs" title="Link to this heading">#</a></h1>
<p>A codec is what lets your objects survive being serialized into a plan and
rebuilt somewhere else — another process, or another program. If your library
contributes table providers, functions, or execution plan nodes and those plans
have to leave the process, you need one.</p>
<p>Codecs are contributed to a session either through an
<a class="reference internal" href="bundles.html#extension-bundles"><span class="std std-ref">extension bundle</span></a>, which is the recommended route,
or one at a time through
<code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code> and
<code class="xref py py-meth docutils literal notranslate"><span class="pre">with_physical_extension_codec()</span></code>.</p>
<section id="codecs-compose">
<h2>Codecs compose<a class="headerlink" href="#codecs-compose" title="Link to this heading">#</a></h2>
<p>Each call to <code class="docutils literal notranslate"><span class="pre">with_logical_extension_codec</span></code> or
<code class="docutils literal notranslate"><span class="pre">with_physical_extension_codec</span></code> <strong>appends</strong> the codec to the session’s codec
chain rather than replacing prior codecs. One session can therefore carry
codecs from several independent libraries at once.</p>
<p><strong>Nothing is asked of the codec itself.</strong> Implement <code class="docutils literal notranslate"><span class="pre">LogicalExtensionCodec</span></code> or
<code class="docutils literal notranslate"><span class="pre">PhysicalExtensionCodec</span></code> exactly as you would for a session that installs only
yours. When your codec writes bytes into a serialized plan, datafusion-python
records which codec wrote them, and strips that record off again before handing
the bytes back. So your codec receives, byte for byte, the payload it wrote,
and is never offered a payload another codec wrote.</p>
<p>A codec that also ships to hosts which dispatch differently may still want its
own guard against foreign payloads. Keeping one is fine; it is simply not
needed for the datafusion-python path.</p>
</section>
<section id="encode-metadata-not-a-handle-to-a-live-object">
<span id="extension-codec-durable-metadata"></span><h2>Encode metadata, not a handle to a live object<a class="headerlink" href="#encode-metadata-not-a-handle-to-a-live-object" title="Link to this heading">#</a></h2>
<p>Your payload has to be enough to rebuild the object somewhere your process is
not. Write the metadata a fresh instance can be constructed from — a path, a
connection string, a schema, the options the object was created with.</p>
<p>The example codecs in this repository do not do this, and it is worth knowing
before copying them. They keep a process-local <code class="docutils literal notranslate"><span class="pre">HashMap</span></code> of live providers and
encode an integer token into it: encoding inserts, decoding removes. That makes
Rust type identity observable across three separately loaded libraries in one
test, which is what the examples exist to show. It also means a decode consumes
its token, so the same bytes cannot be decoded twice, one encoded plan cannot
fan out to several readers, and a plan that never reaches a decoder keeps its
provider alive for the life of the process. A real codec has none of those
properties because it does not park the object anywhere.</p>
</section>
<section id="codec-ids">
<span id="extension-codec-ids"></span><h2>Codec ids<a class="headerlink" href="#codec-ids" title="Link to this heading">#</a></h2>
<p>That record is the codec’s <strong>id</strong>: a short string stored inside the plan,
naming the codec that wrote each payload. Because plans are decoded in another
process — or another program — the id has to name the same codec there as it
did where the plan was written.</p>
<p>Ids are assigned for you. A codec’s id is normally its exporting class’s import
path, such as <code class="docutils literal notranslate"><span class="pre">my_library.Codec</span></code>, which is what you will see in
<code class="xref py py-meth docutils literal notranslate"><span class="pre">logical_extension_codec_ids()</span></code> and in decode
errors. You choose one yourself in three cases:</p>
<ul>
<li><p><strong>Two instances of one class.</strong> Both get the same id, so the second install
raises <code class="docutils literal notranslate"><span class="pre">ValueError</span></code>. Pass <code class="docutils literal notranslate"><span class="pre">codec_id=</span></code> to tell them apart.</p></li>
<li><p><strong>A bare <code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code>.</strong> A capsule has no class to take a name from, so
installing one through <code class="docutils literal notranslate"><span class="pre">with_logical_extension_codec</span></code> or
<code class="docutils literal notranslate"><span class="pre">with_physical_extension_codec</span></code> gives it an id private to the session that
installed it; plans it encodes fail with a clear error on any other session
rather than being decoded by the wrong codec. Pass <code class="docutils literal notranslate"><span class="pre">codec_id=</span></code> if those plans
have to cross sessions.</p>
<p><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_extensions()</span></code> takes no <code class="docutils literal notranslate"><span class="pre">codec_id=</span></code>,
so it refuses a bare capsule outright and tells you to wrap it. See
<a class="reference internal" href="bundles.html#extension-bundles-codecs-are-objects"><span class="std std-ref">Codecs are objects, not capsules</span></a>.</p>
</li>
<li><p><strong>A class you intend to rename.</strong> The id follows the class name, so renaming
stops older plans from decoding. Declare <code class="docutils literal notranslate"><span class="pre">__datafusion_codec_id__</span></code> on the
exporting object to pin an id that survives the rename.</p></li>
</ul>
<p><code class="xref py py-meth docutils literal notranslate"><span class="pre">logical_extension_codec_ids()</span></code> and its
physical counterpart list the ids installed on a session, which is also what a
decode failure names.</p>
<p>Installing one context’s codec stack on another session composes the two
sessions rather than copying codecs out of one: the imported codecs resolve
their task context against the original and stop working when it is dropped —
see <a class="reference internal" href="../contributor-guide/ffi-internals.html#ffi-internals-one-arc"><span class="std std-ref">One session, one Arc&lt;SessionContext&gt;</span></a>. Pass the context itself rather than the
capsule it exports, so its codecs get an id that other sessions can decode.</p>
</section>
<section id="functions-whose-name-is-the-whole-encoding">
<h2>Functions whose name is the whole encoding<a class="headerlink" href="#functions-whose-name-is-the-whole-encoding" title="Link to this heading">#</a></h2>
<p>A codec may own functions that need no payload at all, where the name is the
whole encoding: <code class="docutils literal notranslate"><span class="pre">try_encode_udf</span></code> writes nothing and <code class="docutils literal notranslate"><span class="pre">try_decode_udf</span></code> rebuilds
the function from <code class="docutils literal notranslate"><span class="pre">name</span></code>. That is supported and needs no id, because an <code class="docutils literal notranslate"><span class="pre">Ok</span></code>
with an empty buffer is read as “no opinion” and passes the object to the next
codec. <code class="docutils literal notranslate"><span class="pre">NameOnlyUdfCodec</span></code> in <a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example"><code class="docutils literal notranslate"><span class="pre">datafusion-ffi-example</span></code></a> is the worked case.
Anything no installed codec claims falls through to
<code class="docutils literal notranslate"><span class="pre">Default{Logical,Physical}ExtensionCodec</span></code>.</p>
<p>This is the one case where your decoder is consulted about something you may
not own, because an empty payload has no id to route on. <code class="docutils literal notranslate"><span class="pre">try_decode_udf</span></code> and
its aggregate and window siblings can therefore be called with an empty <code class="docutils literal notranslate"><span class="pre">buf</span></code>
and a <code class="docutils literal notranslate"><span class="pre">name</span></code> belonging to another library. Decide from <code class="docutils literal notranslate"><span class="pre">name</span></code> and return an
error if it is not yours; do not assume <code class="docutils literal notranslate"><span class="pre">buf</span></code> is non-empty.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The framing itself — how an id is stored alongside a payload and routed back,
and the two cases that stay unframed — is internal to datafusion-python and
documented in <code class="docutils literal notranslate"><span class="pre">crates/core/src/codec.rs</span></code> for anyone changing it.</p>
</div>
<p>The current FFI logical codec supports providers and UDFs but not arbitrary
custom <code class="docutils literal notranslate"><span class="pre">LogicalPlan::Extension</span></code> nodes. See both example READMEs for the
supported flow and local build commands.</p>
</section>
<section id="when-codec-order-matters">
<span id="extension-codec-order"></span><h2>When codec order matters<a class="headerlink" href="#when-codec-order-matters" title="Link to this heading">#</a></h2>
<p>Decoding is never order-dependent: a payload names its codec by id and the
chain dispatches straight to it. Encoding walks the chain in install order and
stops at the first codec that claims the node. Most of the time that is
invisible, because libraries claim disjoint things — one owns its table
providers, another its UDFs, a third its own execution plan nodes.</p>
<p>It stops being invisible when a codec claims <em>broadly</em>. A node that came from
another library arrives as an opaque <code class="docutils literal notranslate"><span class="pre">ForeignExecutionPlan</span></code>, and a codec that
claims any of those will take nodes it does not own from any library installed
after it. The query still succeeds. What changes is which library wrote the
bytes — so a plan that has to decode in another process now needs whichever
library happened to win, not the one whose node it is.
<code class="docutils literal notranslate"><span class="pre">MyPhysicalExtensionCodec</span></code> in <a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example"><code class="docutils literal notranslate"><span class="pre">datafusion-ffi-example</span></code></a> claims this way, and
the query-planner example’s test suite pins the consequence.</p>
<p>Two rules of thumb:</p>
<ul class="simple">
<li><p><strong>Writing a codec, claim narrowly.</strong> Downcast to your own types. Claiming a
broad category makes your library order-sensitive for everyone downstream of
it.</p></li>
<li><p><strong>Shipping plans out of the process, verify.</strong> Do not assume your node
reached your codec just because both are installed. Round-trip a plan through
<a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.ExecutionPlan.to_bytes" title="datafusion.ExecutionPlan.to_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">ExecutionPlan.to_bytes</span></code></a> /
<a class="reference internal" href="../autoapi/datafusion/index.html#datafusion.ExecutionPlan.from_bytes" title="datafusion.ExecutionPlan.from_bytes"><code class="xref py py-meth docutils literal notranslate"><span class="pre">from_bytes()</span></code></a> in a test and assert your
codec did the work.</p></li>
</ul>
</section>
<section id="a-codec-decodes-against-the-session-running-the-query">
<span id="extension-codec-decode-session"></span><h2>A codec decodes against the session running the query<a class="headerlink" href="#a-codec-decodes-against-the-session-running-the-query" title="Link to this heading">#</a></h2>
<p>Because the task-context provider comes from the host — see
<a class="reference internal" href="capsule-protocol.html#extension-getter-argument"><span class="std std-ref">What your getter receives</span></a> — a decode callback running inside an
extension library resolves names against the session running the query. A
function registered with <code class="docutils literal notranslate"><span class="pre">ctx.register_udf(...)</span></code> is visible to a foreign codec
decoding a node that references it by name, and the handle is live rather than
a snapshot, so a registration made after the codec is installed is visible too.</p>
<p>This is covered in
<code class="docutils literal notranslate"><span class="pre">examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py</span></code>,
where the example codecs take a <code class="docutils literal notranslate"><span class="pre">require_udf_on_decode</span></code> name and resolve it out
of the task context they are handed.</p>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="functions.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Functions and table functions</p>
</div>
</a>
<a class="right-next"
href="bundles.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">Extension bundles</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#codecs-compose">Codecs compose</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#encode-metadata-not-a-handle-to-a-live-object">Encode metadata, not a handle to a live object</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#codec-ids">Codec ids</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#functions-whose-name-is-the-whole-encoding">Functions whose name is the whole encoding</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#when-codec-order-matters">When codec order matters</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#a-codec-decodes-against-the-session-running-the-query">A codec decodes against the session running the query</a></li>
</ul>
</nav></div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<!-- Based on pydata_sphinx_theme/footer.html -->
<footer class="footer mt-5 mt-md-0">
<div class="container">
<div class="footer-item">
<p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p>
<p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</div>
</footer>
</body>
</html>