Google Git
Sign in
apache/paimon-cpp/refs/heads/gh-pages/./user_guide/read.html
blob: 515f3a55a7e7b82dedbfbeca0224fa87c658355a [file]
<!DOCTYPE html>
<html lang="en" data-content_root="../" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Read &#8212; Apache Paimon C++</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" />
<link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=d7227091" />
<!-- So that users can add custom icons -->
<script defer src="../_static/scripts/fontawesome.js?digest=90905a2f556bf617f1a9"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9" />
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=3bb21c8c"></script>
<script src="../_static/design-tabs.js?v=f930bc37"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'user_guide/read';</script>
<script>DOCUMENTATION_OPTIONS.search_as_you_type = false;</script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Data Cleanup" href="clean.html" />
<link rel="prev" title="Compaction" href="compaction.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="0.4.0" />
<script src="../_static/searchtools.js"></script>
<script src="../_static/language_data.js"></script>
<script src="../searchindex.js"></script>
</head>
<body data-default-mode="">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header id="pst-header" class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class=" navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<p class="title logo__title">Apache Paimon C++</p>
</a></div>
</div>
<div class=" navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../getting_started.html">
Getting Started
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../user_guide.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../examples/index.html">
Examples
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../api.html">
API Reference
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item">
<div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
<i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
</button>
<ul class="dropdown-menu dropdown-menu-end">
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
</ul>
</div></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
</li>
</ul></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../getting_started.html">
Getting Started
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="../user_guide.html">
User Guide
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../examples/index.html">
Examples
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../api.html">
API Reference
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item">
<div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
<i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
</button>
<ul class="dropdown-menu dropdown-menu-end">
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
<li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
</ul>
</div></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
</li>
</ul></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item pst-sidebar-collapse"><button id="pst-collapse-sidebar-button" aria-expanded="true" aria-controls="pst-primary-sidebar">
<svg class="pst-icon" role="img" aria-hidden="true" focusable="false" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg">
<path fill="currentColor" d="M3 15.5C2.36232 15.5 1.74874 15.2564 1.28478 14.8189C0.820828 14.3815 0.541576 13.7832 0.504167 13.1467L0.5 13L0.5 3C0.499965 2.36232 0.743605 1.74874 1.18107 1.28478C1.61854 0.820828 2.21676 0.541576 2.85333 0.504167L3 0.5L13 0.5C13.6377 0.499965 14.2513 0.743605 14.7152 1.18107C15.1792 1.61854 15.4584 2.21676 15.4958 2.85333L15.5 3L15.5 13C15.5 13.6377 15.2564 14.2513 14.8189 14.7152C14.3815 15.1792 13.7832 15.4584 13.1467 15.4958L13 15.5L3 15.5ZM3 13.8333L10.5 13.8333L10.5 2.16667L3 2.16667C2.79589 2.16669 2.59889 2.24163 2.44636 2.37726C2.29383 2.5129 2.19638 2.69979 2.1725 2.9025L2.16667 3L2.16667 13C2.16669 13.2041 2.24163 13.4011 2.37726 13.5536C2.5129 13.7062 2.69979 13.8036 2.9025 13.8275L3 13.8333ZM6.65583 10.325L6.5775 10.2558L4.91083 8.58917C4.76735 8.44567 4.68116 8.25476 4.66843 8.05223C4.65569 7.84971 4.71729 7.6495 4.84167 7.48917L4.91083 7.41083L6.5775 5.74417C6.72747 5.59471 6.9287 5.50794 7.14032 5.50148C7.35194 5.49502 7.55809 5.56935 7.7169 5.70937C7.8757 5.8494 7.97525 6.04463 7.99533 6.25539C8.01541 6.46616 7.95451 6.67667 7.825 6.84417L7.75583 6.9225L6.67917 8L7.75583 9.0775C7.89931 9.22099 7.98551 9.41191 7.99824 9.61443C8.01097 9.81695 7.94938 10.0172 7.825 10.1775L7.75583 10.2558C7.61234 10.3993 7.42142 10.4855 7.2189 10.4982C7.01638 10.511 6.81617 10.4494 6.65583 10.325Z"/>
</svg>
<span class="pst-collapse-sidebar-label">Collapse Sidebar</span>
<span class="pst-expand-sidebar-label">Expand Sidebar</span>
</button></div>
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="catalog.html">Catalog</a></li>
<li class="toctree-l1"><a class="reference internal" href="schema.html">Schema</a></li>
<li class="toctree-l1"><a class="reference internal" href="snapshot.html">Snapshot</a></li>
<li class="toctree-l1"><a class="reference internal" href="manifest.html">Manifest</a></li>
<li class="toctree-l1"><a class="reference internal" href="manifest_cache.html">Manifest Cache</a></li>
<li class="toctree-l1"><a class="reference internal" href="manifest_entry_cache.html">Manifest Entry Cache</a></li>
<li class="toctree-l1"><a class="reference internal" href="parquet_metadata_cache.html">Parquet Metadata Cache</a></li>
<li class="toctree-l1"><a class="reference internal" href="data_types.html">Data Types</a></li>
<li class="toctree-l1"><a class="reference internal" href="primary_key_table.html">Primary Key Table</a></li>
<li class="toctree-l1"><a class="reference internal" href="append_only_table.html">Append Only Table</a></li>
<li class="toctree-l1"><a class="reference internal" href="system_tables.html">System Tables</a></li>
<li class="toctree-l1"><a class="reference internal" href="write.html">Write</a></li>
<li class="toctree-l1"><a class="reference internal" href="commit.html">Commit</a></li>
<li class="toctree-l1"><a class="reference internal" href="compaction.html">Compaction</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Read</a></li>
<li class="toctree-l1"><a class="reference internal" href="clean.html">Data Cleanup</a></li>
<li class="toctree-l1"><a class="reference internal" href="prefetch.html">Prefetch</a></li>
<li class="toctree-l1"><a class="reference internal" href="arrow.html">Memory Format</a></li>
<li class="toctree-l1"><a class="reference internal" href="global_index.html">Global Index</a></li>
<li class="toctree-l1"><a class="reference internal" href="primary_key_global_index.html">Primary Key Global Index</a></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../user_guide.html" class="nav-link">User Guide</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Read</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<section id="read">
<h1>Read<a class="headerlink" href="#read" title="Link to this heading">#</a></h1>
<p>Paimon by functionality can be divided into two layers:</p>
<ul class="simple">
<li><p>Control Plane: Responsible for accessing and managing Meta (snapshot, manifest, etc.), including:
- Catalog / Database access
- Table retrieval
- Collection and resolution of data files</p></li>
<li><p>Data Plane: Responsible for accessing actual data files, including:
- Readers for various file formats
- Coordinated reading of file collections</p></li>
</ul>
<p>The control plane and data plane interact primarily via DataSplit (the query plan). Paimon C++ currently supports a standard
DataSplit protocol which includes the necessary meta information to access data files. With DataSplit, a high-performance
data access path can be integrated.</p>
<p>At compute time, the execution engine (reader) does not need to be aware of the concrete table type or its metadata details.
It only needs to follow the instructions within the DataSplit (query plan) to perform data reading operations.</p>
<p>With the layered abstraction of the control plane and data plane, and the use of DataSplit as a stable protocol interface,
the two layers can evolve their functionality and optimize code relatively independently. This design also enables
cross-language task scheduling and interaction (e.g., Java and C++), substantially reducing engineering maintenance costs
across the two language ecosystems.</p>
<section id="schema-evolution">
<h2>Schema Evolution<a class="headerlink" href="#schema-evolution" title="Link to this heading">#</a></h2>
<section id="scope-and-compatibility">
<h3>Scope and Compatibility<a class="headerlink" href="#scope-and-compatibility" title="Link to this heading">#</a></h3>
<p>Paimon C++ supports all evolution kinds available in Java Paimon for non-nested types:</p>
<ul class="simple">
<li><p>Add column</p></li>
<li><p>Drop column</p></li>
<li><p>Reorder columns</p></li>
<li><p>Rename column</p></li>
<li><p>Change column type</p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<ul class="simple">
<li><p>Only non-nested type evolution is supported. Nested columns (struct, array, map) are not supported.</p></li>
<li><p>Partition keys: Only column reordering is supported; other operations are not supported (consistent with Java Paimon).</p></li>
<li><p>Primary key:</p>
<ul>
<li><p>Adding or dropping columns is not supported.</p></li>
<li><p>Other operations are supported (consistent with Java Paimon).</p></li>
</ul>
</li>
</ul>
</div>
</section>
<section id="per-file-schema-via-field-ids">
<h3>Per-File Schema via Field IDs<a class="headerlink" href="#per-file-schema-via-field-ids" title="Link to this heading">#</a></h3>
<p>In DataSplit, each file may have a completely different data schema. Paimon uses field IDs to uniquely identify fields.</p>
</section>
<section id="overflow-behavior-disclaimer">
<h3>Overflow Behavior Disclaimer<a class="headerlink" href="#overflow-behavior-disclaimer" title="Link to this heading">#</a></h3>
<p>Overflow behavior is undefined for C++ and Java Paimon. Results in overflow scenarios may:</p>
<ul class="simple">
<li><p>Be incorrect values,</p></li>
<li><p>Return an error status,</p></li>
<li><p>Or be null.</p></li>
</ul>
<p>Paimon C++ does not guarantee identical results to Java Paimon in overflow scenarios. Users should not rely on identical
return values between implementations.</p>
<p>One exception: casting <code class="docutils literal notranslate"><span class="pre">float</span></code> or <code class="docutils literal notranslate"><span class="pre">double</span></code> to an integer type is well defined and matches Java on every supported
architecture; see note 2️⃣ below the matrix.</p>
</section>
<section id="type-change-support-matrix">
<h3>Type Change Support Matrix<a class="headerlink" href="#type-change-support-matrix" title="Link to this heading">#</a></h3>
<p>The table below indicates support for changing a column type from <code class="docutils literal notranslate"><span class="pre">source</span></code> to <code class="docutils literal notranslate"><span class="pre">target</span></code>. Refer to the numbered notes below the table
for caveats.</p>
<div class="pst-scrollable-table-container"><table class="table">
<colgroup>
<col style="width: 8.7%" />
<col style="width: 7.2%" />
<col style="width: 7.2%" />
<col style="width: 7.2%" />
<col style="width: 7.2%" />
<col style="width: 7.2%" />
<col style="width: 7.2%" />
<col style="width: 5.8%" />
<col style="width: 8.7%" />
<col style="width: 7.2%" />
<col style="width: 5.8%" />
<col style="width: 13.0%" />
<col style="width: 7.2%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>src \ target</p></th>
<th class="head"><p>tinyint</p></th>
<th class="head"><p>smallint</p></th>
<th class="head"><p>int</p></th>
<th class="head"><p>bigint</p></th>
<th class="head"><p>float</p></th>
<th class="head"><p>double</p></th>
<th class="head"><p>bool</p></th>
<th class="head"><p>string</p></th>
<th class="head"><p>binary</p></th>
<th class="head"><p>date</p></th>
<th class="head"><p>timestamp (without tz)</p></th>
<th class="head"><p>decimal</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>tinyint</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
</tr>
<tr class="row-odd"><td><p>smallint</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
</tr>
<tr class="row-even"><td><p>int</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅</p></td>
</tr>
<tr class="row-odd"><td><p>bigint</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅ 6️⃣</p></td>
<td><p>✅</p></td>
</tr>
<tr class="row-even"><td><p>float</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅ 3️⃣ 4️⃣</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
</tr>
<tr class="row-odd"><td><p>double</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅ 2️⃣</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅ 3️⃣ 4️⃣</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
</tr>
<tr class="row-even"><td><p>bool</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
</tr>
<tr class="row-odd"><td><p>string</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅ 3️⃣</p></td>
<td><p>✅ 3️⃣</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>✅ 5️⃣</p></td>
<td><p>✅ 7️⃣</p></td>
</tr>
<tr class="row-even"><td><p>binary</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
</tr>
<tr class="row-odd"><td><p>date</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
<td><p>✅ 5️⃣</p></td>
<td><p>❌</p></td>
</tr>
<tr class="row-even"><td><p>timestamp (without tz)</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
</tr>
<tr class="row-odd"><td><p>decimal</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅ 1️⃣</p></td>
<td><p>✅</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>❌</p></td>
<td><p>✅</p></td>
</tr>
</tbody>
</table>
</div>
<div class="note admonition">
<p class="admonition-title">Overflow Behavior Notes</p>
<dl>
<dt>1️⃣ Integer downcast overflow behavior matches Java in specific cases.</dt><dd><p>Example: smallint -&gt; tinyint, 32767 becomes -1; int -&gt; smallint, -2147483648 becomes 0.</p>
</dd>
<dt>2️⃣ Casting float/double to an integer type follows Java semantics on every supported architecture:</dt><dd><p><code class="docutils literal notranslate"><span class="pre">NaN</span></code> becomes 0, an out-of-range value saturates at the int32 bounds (int64 for <code class="docutils literal notranslate"><span class="pre">bigint</span></code>),
and the result is narrowed to the target width by keeping the low bits, the way Java
narrowing does.</p>
<p>Example: float -&gt; tinyint, C++ and Java both: <code class="docutils literal notranslate"><span class="pre">MAX_FLOAT</span> <span class="pre">-&gt;</span> <span class="pre">-1</span></code>, <code class="docutils literal notranslate"><span class="pre">INFINITY</span> <span class="pre">-&gt;</span> <span class="pre">-1</span></code>,
<code class="docutils literal notranslate"><span class="pre">NaN</span> <span class="pre">-&gt;</span> <span class="pre">0</span></code>, <code class="docutils literal notranslate"><span class="pre">300.9</span> <span class="pre">-&gt;</span> <span class="pre">44</span></code>.</p>
<p>double -&gt; float overflow produces <code class="docutils literal notranslate"><span class="pre">Infinity</span></code> / <code class="docutils literal notranslate"><span class="pre">-Infinity</span></code>, consistent with Java.</p>
</dd>
<dt>3️⃣ Keyword differences for special float/double values:</dt><dd><ul class="simple">
<li><p>Java: Infinity, -Infinity, NaN</p></li>
<li><p>C++: inf, -inf, nan</p></li>
</ul>
</dd>
<dt>4️⃣ Printing difference:</dt><dd><ul class="simple">
<li><p>C++ prints 1.0 as <code class="docutils literal notranslate"><span class="pre">1</span></code></p></li>
<li><p>Java prints 1.0 as <code class="docutils literal notranslate"><span class="pre">1.0</span></code></p></li>
</ul>
</dd>
<dt>5️⃣ Timestamp precision and range differences:</dt><dd><ul class="simple">
<li><p>Java: 0000-01-01 00:00:00.000000000 to 9999-12-31 23:59:59.999999999</p></li>
<li><p>C++: 1677-09-21 00:12:43.145224192 to 2262-04-11 23:47:16.854775807</p></li>
<li><p>C++ only supports nanosecond precision; range is smaller.</p></li>
</ul>
</dd>
<dt>6️⃣ bigint -&gt; timestamp range differences:</dt><dd><ul class="simple">
<li><p>Java (ms): <code class="docutils literal notranslate"><span class="pre">[MIN_INT64/1000,</span> <span class="pre">MAX_INT64/1000]</span></code> seconds</p></li>
<li><p>C++ (ns): <code class="docutils literal notranslate"><span class="pre">[MIN_INT64/1e9,</span>&#160; <span class="pre">MAX_INT64/1e9]</span></code> seconds</p></li>
</ul>
</dd>
<dt>7️⃣ string -&gt; decimal with precision &gt; 38:</dt><dd><ul class="simple">
<li><p>C++ returns <code class="docutils literal notranslate"><span class="pre">null</span></code> if parsing would overflow 128-bit arithmetic.</p></li>
<li><p>Java may rescale and return a value based on the rescaled precision.</p></li>
<li><p>Example input: <code class="docutils literal notranslate"><span class="pre">1111111111111111111111111111111111111.15</span></code>, Java returns: <code class="docutils literal notranslate"><span class="pre">1111111111111111111111111111111111111.2</span></code>, C++ returns: <code class="docutils literal notranslate"><span class="pre">null</span></code></p></li>
</ul>
</dd>
</dl>
</div>
</section>
<section id="implementation-guidance">
<h3>Implementation Guidance<a class="headerlink" href="#implementation-guidance" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p>Use DataSplit as the sole interface between control and data planes. Treat it as the canonical query plan contract.</p></li>
<li><p>Resolve field types and IDs per file; prefer inline data file metadata, fallback to table schema files when necessary.</p></li>
<li><p>Expect per-file schema variability; design readers to align by field IDs rather than positional indices.</p></li>
<li><p>Do not assume identical overflow semantics across C++ and Java; tests should validate acceptable ranges and nullability.</p></li>
<li><p>For timestamp handling, consider precision/range constraints in C++ when interoperating with Java-produced data splits.</p></li>
</ul>
</section>
</section>
<section id="deletion-vectors-on-data-evolution-tables">
<span id="data-evolution-deletion-vectors"></span><h2>Deletion Vectors on Data-Evolution Tables<a class="headerlink" href="#deletion-vectors-on-data-evolution-tables" title="Link to this heading">#</a></h2>
<p>A data-evolution table (<code class="docutils literal notranslate"><span class="pre">data-evolution.enabled</span> <span class="pre">=</span> <span class="pre">true</span></code>) may enable
<code class="docutils literal notranslate"><span class="pre">deletion-vectors.enabled</span></code>. Reading such a table is supported: a deleted row disappears
from the result, including from the columns merged out of the other files that cover it.</p>
<section id="how-the-deletion-vector-is-located">
<h3>How the Deletion Vector Is Located<a class="headerlink" href="#how-the-deletion-vector-is-located" title="Link to this heading">#</a></h3>
<p>A data-evolution split holds several files per row id range, one per group of columns. The
deletion vector of such a row range group is not per file: it is maintained against the
group’s <em>anchor file</em>, the oldest normal file of the group, compared by
<code class="docutils literal notranslate"><span class="pre">(max_sequence_number,</span> <span class="pre">file_name)</span></code> and skipping blob and vector-store files. Its positions
are therefore relative to the anchor file’s row id range.</p>
<p>Reading applies that one vector to every file of the group, shifted by the file’s offset
inside the anchor range, so all the readers being merged drop the same rows and stay
positionally aligned. The blob fallback path has no file reader to wrap for the placeholder
gaps it pads uncovered row ids with, so it removes the deleted row ids from those gap ranges
instead.</p>
<p>The rule that picks the anchor has to stay identical to the engine that writes the vectors:
a vector keyed by any other file of the group is never found, and its deleted rows silently
come back.</p>
</section>
<section id="limitations">
<h3>Limitations<a class="headerlink" href="#limitations" title="Link to this heading">#</a></h3>
<ul class="simple">
<li><p>Only the default 32-bit deletion vectors can be read. <code class="docutils literal notranslate"><span class="pre">deletion-vectors.bitmap64</span></code> is not
supported yet, and a read fails when it actually encounters a 64-bit deletion vector.</p></li>
<li><p>Paimon C++ does not write deletion vectors for data-evolution tables, so the deletes
themselves have to be issued by another engine.</p></li>
<li><p>A commit that drops data files from such a table, an overwrite for instance, is refused.
Whether it conflicts with a concurrent commit rewriting those files’ deletion vectors cannot
be decided yet, so it fails rather than committing against a stale state. Appending is
unaffected, and so is another engine replacing a deletion vector.</p></li>
<li><p>Such a table is never compacted; see
<a class="reference internal" href="compaction.html#data-evolution-deletion-vectors-compaction"><span class="std std-ref">the compaction note</span></a>.</p></li>
</ul>
</section>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="compaction.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Compaction</p>
</div>
</a>
<a class="right-next"
href="clean.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">Data Cleanup</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav id="pst-page-toc-nav" class="page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="pst-show_toc_level nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#schema-evolution">Schema Evolution</a><ul class="pst-show_toc_level nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#scope-and-compatibility">Scope and Compatibility</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#per-file-schema-via-field-ids">Per-File Schema via Field IDs</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#overflow-behavior-disclaimer">Overflow Behavior Disclaimer</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#type-change-support-matrix">Type Change Support Matrix</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guidance">Implementation Guidance</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#deletion-vectors-on-data-evolution-tables">Deletion Vectors on Data-Evolution Tables</a><ul class="pst-show_toc_level nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#how-the-deletion-vector-is-located">How the Deletion Vector Is Located</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#limitations">Limitations</a></li>
</ul>
</li>
</ul>
</nav></div>
<div class="sidebar-secondary-item">
<div class="tocsection editthispage">
<a href="https://github.com/apache/paimon-cpp/edit/main/docs/source/user_guide/read.rst">
<i class="fa-solid fa-pencil"></i>
Edit on GitHub
</a>
</div>
</div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9"></script>
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 2026-2026 Apache Software Foundation.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 9.1.0.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
<!-- # L10n: Setting the PST URL as an argument as this does not need to be localized -->
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.20.0.
</p></div>
</div>
</div>
</footer>
</body>
</html>