Google Git
Sign in
apache/paimon-cpp/refs/heads/gh-pages/./user_guide/arrow.html
blob: ceb0b2b970d1c0f29d180635697b275082f9dfa8 [file]
<!DOCTYPE html>
<html lang="en" data-content_root="../" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Memory Format &#8212; C++ Paimon</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" />
<link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=d7227091" />
<!-- So that users can add custom icons -->
<script defer src="../_static/scripts/fontawesome.js?digest=90905a2f556bf617f1a9"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9" />
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=3bb21c8c"></script>
<script src="../_static/design-tabs.js?v=f930bc37"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'user_guide/arrow';</script>
<script>DOCUMENTATION_OPTIONS.search_as_you_type = false;</script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Global Index" href="global_index.html" />
<link rel="prev" title="Prefetch" href="prefetch.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="0.2.3" />
<script src="../_static/searchtools.js"></script>
<script src="../_static/language_data.js"></script>
<script src="../searchindex.js"></script>
</head>
<body data-default-mode="">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header id="pst-header" class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class=" navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<p class="title logo__title">Paimon C++</p>
</a></div>
</div>
<div class=" navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../getting_started.html">
Getting Started
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../user_guide.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../examples/index.html">
Examples
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../api.html">
API Reference
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item">
<div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
<i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
</button>
<ul class="dropdown-menu dropdown-menu-end">
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
</ul>
</div></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
</li>
</ul></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../getting_started.html">
Getting Started
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../user_guide.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../examples/index.html">
Examples
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../api.html">
API Reference
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item">
<div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
<i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
</button>
<ul class="dropdown-menu dropdown-menu-end">
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
</ul>
</div></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
</li>
</ul></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item pst-sidebar-collapse"><button id="pst-collapse-sidebar-button" aria-expanded="true" aria-controls="pst-primary-sidebar">
<svg class="pst-icon" role="img" aria-hidden="true" focusable="false" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg">
<path fill="currentColor" d="M3 15.5C2.36232 15.5 1.74874 15.2564 1.28478 14.8189C0.820828 14.3815 0.541576 13.7832 0.504167 13.1467L0.5 13L0.5 3C0.499965 2.36232 0.743605 1.74874 1.18107 1.28478C1.61854 0.820828 2.21676 0.541576 2.85333 0.504167L3 0.5L13 0.5C13.6377 0.499965 14.2513 0.743605 14.7152 1.18107C15.1792 1.61854 15.4584 2.21676 15.4958 2.85333L15.5 3L15.5 13C15.5 13.6377 15.2564 14.2513 14.8189 14.7152C14.3815 15.1792 13.7832 15.4584 13.1467 15.4958L13 15.5L3 15.5ZM3 13.8333L10.5 13.8333L10.5 2.16667L3 2.16667C2.79589 2.16669 2.59889 2.24163 2.44636 2.37726C2.29383 2.5129 2.19638 2.69979 2.1725 2.9025L2.16667 3L2.16667 13C2.16669 13.2041 2.24163 13.4011 2.37726 13.5536C2.5129 13.7062 2.69979 13.8036 2.9025 13.8275L3 13.8333ZM6.65583 10.325L6.5775 10.2558L4.91083 8.58917C4.76735 8.44567 4.68116 8.25476 4.66843 8.05223C4.65569 7.84971 4.71729 7.6495 4.84167 7.48917L4.91083 7.41083L6.5775 5.74417C6.72747 5.59471 6.9287 5.50794 7.14032 5.50148C7.35194 5.49502 7.55809 5.56935 7.7169 5.70937C7.8757 5.8494 7.97525 6.04463 7.99533 6.25539C8.01541 6.46616 7.95451 6.67667 7.825 6.84417L7.75583 6.9225L6.67917 8L7.75583 9.0775C7.89931 9.22099 7.98551 9.41191 7.99824 9.61443C8.01097 9.81695 7.94938 10.0172 7.825 10.1775L7.75583 10.2558C7.61234 10.3993 7.42142 10.4855 7.2189 10.4982C7.01638 10.511 6.81617 10.4494 6.65583 10.325Z"/>
</svg>
<span class="pst-collapse-sidebar-label">Collapse Sidebar</span>
<span class="pst-expand-sidebar-label">Expand Sidebar</span>
</button></div>
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="catalog.html">Catalog</a></li>
<li class="toctree-l1"><a class="reference internal" href="schema.html">Schema</a></li>
<li class="toctree-l1"><a class="reference internal" href="snapshot.html">Snapshot</a></li>
<li class="toctree-l1"><a class="reference internal" href="manifest.html">Manifest</a></li>
<li class="toctree-l1"><a class="reference internal" href="data_types.html">Data Types</a></li>
<li class="toctree-l1"><a class="reference internal" href="primary_key_table.html">Primary Key Table</a></li>
<li class="toctree-l1"><a class="reference internal" href="append_only_table.html">Append Only Table</a></li>
<li class="toctree-l1"><a class="reference internal" href="write.html">Write</a></li>
<li class="toctree-l1"><a class="reference internal" href="commit.html">Commit</a></li>
<li class="toctree-l1"><a class="reference internal" href="compaction.html">Compaction</a></li>
<li class="toctree-l1"><a class="reference internal" href="read.html">Read</a></li>
<li class="toctree-l1"><a class="reference internal" href="clean.html">Data Cleanup</a></li>
<li class="toctree-l1"><a class="reference internal" href="prefetch.html">Prefetch</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Memory Format</a></li>
<li class="toctree-l1"><a class="reference internal" href="global_index.html">Global Index</a></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../user_guide.html" class="nav-link">User Guide</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Memory Format</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<section id="memory-format">
<span id="id1"></span><h1>Memory Format<a class="headerlink" href="#memory-format" title="Link to this heading">#</a></h1>
<p><a class="reference external" href="https://paimon.apache.org/docs/master/">Paimon Java</a> uses a row-level
abstraction, <code class="docutils literal notranslate"><span class="pre">InternalRow</span></code>, as the default data format interface.
Row-level interfaces are generally more intuitive for programming and can be
easily integrated into data processing pipelines such as filtering and merge sorting.
However, this approach introduces conversion and access overhead,
making it difficult to fully leverage the additional performance benefits provided
by modern CPU SIMD vectorization.</p>
<p>Considering that the C++ implementation focuses more on end-to-end performance,
and that underlying data file formats (e.g., ORC, Parquet) are predominantly
columnar, providing a columnar-centric data abstraction can minimize overhead
during data flow, integrate more naturally with vectorized execution engines,
and ultimately deliver superior overall performance.</p>
<section id="why-apache-arrow">
<h2>Why Apache Arrow<a class="headerlink" href="#why-apache-arrow" title="Link to this heading">#</a></h2>
<p>Apache Arrow is currently the most widely adopted in-memory columnar format and
has strong native support for Parquet and ORC. It is well-integrated across
open-source engines including Spark, Pandas, Drill, Impala, and Velox.</p>
<p>Overall, using Apache Arrow as the in-memory format for Paimon C++ allows us to:</p>
<ul class="simple">
<li><p>Maximize columnar performance and efficient data access patterns.</p></li>
<li><p>Seamlessly integrate with the Apache Arrow ecosystem and tooling.</p></li>
<li><p>Benefit from mature interoperability with popular data systems and formats.</p></li>
</ul>
</section>
<section id="versioning-and-dependency-concerns">
<h2>Versioning and Dependency Concerns<a class="headerlink" href="#versioning-and-dependency-concerns" title="Link to this heading">#</a></h2>
<p>One important consideration is that Arrow is an active open-source project with
a broad and evolving C++ surface area that spans data formats, compute kernels,
and file I/O. Due to frequent releases and a large API surface, different Arrow
C++ SDK versions can introduce API incompatibilities.</p>
<p>If Paimon C++ directly depends on the full Arrow C++ SDK, it may conflict with
existing Arrow C++ dependencies in other compute engines, raising integration
costs and increasing long-term maintenance complexity.</p>
</section>
<section id="adopting-the-arrow-c-data-interface">
<h2>Adopting the Arrow C Data Interface<a class="headerlink" href="#adopting-the-arrow-c-data-interface" title="Link to this heading">#</a></h2>
<p>To leverage Arrow’s performance and ecosystem benefits while avoiding tight
coupling to specific Arrow C++ SDK versions, we use the Arrow C Data
Interface as the default in-memory format for Paimon C++.</p>
<p>Key advantages:</p>
<ul class="simple">
<li><p>Version-neutral: The C Data Interface is designed to be stable and forward-compatible
across Arrow versions.</p></li>
<li><p>Compiler-neutral: C language interfaces avoid ABI friction commonly seen with
C++ compilers and standard libraries.</p></li>
<li><p>Broad interoperability: The C Data Interface is supported by Arrow-based
systems and enables zero-copy or minimal-copy interchange of columnar data.</p></li>
</ul>
</section>
<section id="design-principles">
<h2>Design Principles<a class="headerlink" href="#design-principles" title="Link to this heading">#</a></h2>
<ul class="simple">
<li><p>Columnar-first abstraction:
Paimon C++ will represent in-memory data using columnar buffers and schemas
compatible with the Arrow C Data Interface, minimizing transformation overhead.</p></li>
<li><p>Minimal dependency footprint:
Prefer stable C interfaces and lightweight utility layers; avoid linking
against the full Arrow C++ SDK unless strictly necessary and well-isolated.</p></li>
<li><p>Vectorization-aware execution:
Structure data layouts to align with SIMD processing (e.g., contiguous
buffers, clear null bitmaps and type-specific arrays), enabling efficient
filtering, projection, and aggregation.</p></li>
<li><p>Interoperability:
Ensure that data produced and consumed by Paimon C++ can be handed off to
Arrow-compatible engines and libraries without expensive conversions.</p></li>
<li><p>Compatibility with columnar storage:
Maintain efficient paths from columnar file formats (Parquet, ORC) to in-memory
columnar representations, minimizing decoding and marshaling overhead.</p></li>
</ul>
</section>
<section id="implementation-outline">
<h2>Implementation Outline<a class="headerlink" href="#implementation-outline" title="Link to this heading">#</a></h2>
<section id="schema-and-buffers">
<h3>Schema and buffers<a class="headerlink" href="#schema-and-buffers" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p>Represent schemas and arrays using Arrow C Data Interface types (e.g., <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code>, <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code>) with clear ownership and lifecycle.</p></li>
<li><p>Support nested types (structs, lists, maps) and common primitives (integers, floats, decimals, timestamps).</p></li>
</ul>
</section>
<section id="memory-management">
<h3>Memory management<a class="headerlink" href="#memory-management" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p>Define consistent ownership semantics for buffers and child arrays.</p></li>
<li><p>Employ reference counting or explicit release callbacks aligned with the Arrow C conventions.</p></li>
</ul>
</section>
<section id="nullability-and-validity">
<h3>Nullability and validity<a class="headerlink" href="#nullability-and-validity" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p>Use standard validity bitmaps for nullability and adhere to Arrow’s canonical buffer organization (validity, offsets, data, etc.).</p></li>
</ul>
</section>
<section id="conversion-boundaries">
<h3>Conversion boundaries<a class="headerlink" href="#conversion-boundaries" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p>Provide adapters to
* Read from columnar file formats (Parquet/ORC) into Arrow-compatible <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structures.
* Export/import data to other Arrow-compatible engines with zero or minimal copies.</p></li>
<li><p>Keep these adapters independent from heavy Arrow C++ SDK dependencies.</p></li>
</ul>
</section>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="prefetch.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Prefetch</p>
</div>
</a>
<a class="right-next"
href="global_index.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">Global Index</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav id="pst-page-toc-nav" class="page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="pst-show_toc_level nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-apache-arrow">Why Apache Arrow</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#versioning-and-dependency-concerns">Versioning and Dependency Concerns</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adopting-the-arrow-c-data-interface">Adopting the Arrow C Data Interface</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#design-principles">Design Principles</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-outline">Implementation Outline</a><ul class="pst-show_toc_level nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#schema-and-buffers">Schema and buffers</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#memory-management">Memory management</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#nullability-and-validity">Nullability and validity</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#conversion-boundaries">Conversion boundaries</a></li>
</ul>
</li>
</ul>
</nav></div>
<div class="sidebar-secondary-item">
<div class="tocsection editthispage">
<a href="https://github.com/apache/paimon-cpp/edit/main/docs/source/user_guide/arrow.rst">
<i class="fa-solid fa-pencil"></i>
Edit on GitHub
</a>
</div>
</div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9"></script>
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 2024-2026 Apache Software Foundation.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 9.1.0.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
<!-- # L10n: Setting the PST URL as an argument as this does not need to be localized -->
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.20.0.
</p></div>
</div>
</div>
</footer>
</body>
</html>