user_guide/write.html - paimon-cpp - Git at Google


 <!DOCTYPE html>


 <html lang="en" data-content_root="../" >

   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />

     <title>Write &#8212; C++ Paimon</title>


   <script data-cfasync="false">
     document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
     document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
   </script>
   <!--
     this give us a css class that will be invisible only if js is disabled
   -->
   <noscript>
     <style>
       .pst-js-only { display: none !important; }

     </style>
   </noscript>

   <!-- Loaded before other Sphinx assets -->
   <link href="../_static/styles/theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />
 <link href="../_static/styles/pydata-sphinx-theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" />

     <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
     <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
     <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" />
     <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=d7227091" />

   <!-- So that users can add custom icons -->
   <script defer src="../_static/scripts/fontawesome.js?digest=90905a2f556bf617f1a9"></script>
   <!-- Pre-loaded scripts that we'll load fully later -->
   <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9" />
 <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9" />

     <script src="../_static/documentation_options.js?v=5929fcd5"></script>
     <script src="../_static/doctools.js?v=fd6eb6e6"></script>
     <script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../_static/clipboard.min.js?v=a7894cd8"></script>
     <script src="../_static/copybutton.js?v=3bb21c8c"></script>
     <script src="../_static/design-tabs.js?v=f930bc37"></script>
     <script>DOCUMENTATION_OPTIONS.pagename = 'user_guide/write';</script>
     <script>DOCUMENTATION_OPTIONS.search_as_you_type = false;</script>
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" />
     <link rel="next" title="Commit" href="commit.html" />
     <link rel="prev" title="Append Only Table" href="append_only_table.html" />
   <meta name="viewport" content="width=device-width, initial-scale=1"/>
   <meta name="docsearch:language" content="en"/>
   <meta name="docsearch:version" content="0.2.3" />


     <script src="../_static/searchtools.js"></script>
     <script src="../_static/language_data.js"></script>
     <script src="../searchindex.js"></script>

   </head>
   <body data-default-mode="">


   <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>


   <div id="pst-scroll-pixel-helper"></div>

   <button type="button" class="btn rounded-pill" id="pst-back-to-top">
     <i class="fa-solid fa-arrow-up"></i>Back to top</button>


   <dialog id="pst-search-dialog">

 <form class="bd-search d-flex align-items-center"
       action="../search.html"
       method="get">
   <i class="fa-solid fa-magnifying-glass"></i>
   <input type="search"
          class="form-control"
          name="q"
          placeholder="Search the docs ..."
          aria-label="Search the docs ..."
          autocomplete="off"
          autocorrect="off"
          autocapitalize="off"
          spellcheck="false"/>
   <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
 </form>
   </dialog>

   <div class="pst-async-banner-revealer d-none">
   <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
 </div>


     <header id="pst-header" class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
 <div class="bd-header__inner bd-page-width">
   <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
     <span class="fa-solid fa-bars"></span>
   </button>


   <div class=" navbar-header-items__start">

       <div class="navbar-item">


 <a class="navbar-brand logo" href="../index.html">


     <p class="title logo__title">Paimon C++</p>

 </a></div>

   </div>

   <div class=" navbar-header-items">

     <div class="me-auto navbar-header-items__center">

         <div class="navbar-item">
 <nav>
   <ul class="bd-navbar-elements navbar-nav">

 <li class="nav-item ">
   <a class="nav-link nav-internal" href="../getting_started.html">
     Getting Started
   </a>
 </li>


 <li class="nav-item current active">
   <a class="nav-link nav-internal" href="../user_guide.html">
     User Guide
   </a>
 </li>


 <li class="nav-item ">
   <a class="nav-link nav-internal" href="../examples/index.html">
     Examples
   </a>
 </li>


 <li class="nav-item ">
   <a class="nav-link nav-internal" href="../api.html">
     API Reference
   </a>
 </li>

   </ul>
 </nav></div>

     </div>


     <div class="navbar-header-items__end">

         <div class="navbar-item navbar-persistent--container">


 <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
  <i class="fa-solid fa-magnifying-glass"></i>
  <span class="search-button__default-text">Search</span>
  <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
 </button>
         </div>


         <div class="navbar-item">

 <div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
   <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
     <i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
     <i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
     <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
   </button>
   <ul class="dropdown-menu dropdown-menu-end">
     <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
     <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
     <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
   </ul>
 </div></div>

         <div class="navbar-item"><ul class="navbar-icon-links"
     aria-label="Icon Links">
         <li class="nav-item">


           <a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
         </li>
 </ul></div>

     </div>

   </div>


     <div class="navbar-persistent--mobile">

 <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
  <i class="fa-solid fa-magnifying-glass"></i>
  <span class="search-button__default-text">Search</span>
  <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
 </button>
     </div>


     <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
       <span class="fa-solid fa-outdent"></span>
     </button>

 </div>

     </header>


   <div class="bd-container">
     <div class="bd-container__inner bd-page-width">


       <dialog id="pst-primary-sidebar-modal"></dialog>
       <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">


   <div class="sidebar-header-items sidebar-primary__section">


       <div class="sidebar-header-items__center">


             <div class="navbar-item">
 <nav>
   <ul class="bd-navbar-elements navbar-nav">

 <li class="nav-item ">
   <a class="nav-link nav-internal" href="../getting_started.html">
     Getting Started
   </a>
 </li>


 <li class="nav-item current active">
   <a class="nav-link nav-internal" href="../user_guide.html">
     User Guide
   </a>
 </li>


 <li class="nav-item ">
   <a class="nav-link nav-internal" href="../examples/index.html">
     Examples
   </a>
 </li>


 <li class="nav-item ">
   <a class="nav-link nav-internal" href="../api.html">
     API Reference
   </a>
 </li>

   </ul>
 </nav></div>


       </div>


       <div class="sidebar-header-items__end">

           <div class="navbar-item">

 <div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode">
   <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown">
     <i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i>
     <i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i>
     <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i>
   </button>
   <ul class="dropdown-menu dropdown-menu-end">
     <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li>
     <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li>
     <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li>
   </ul>
 </div></div>

           <div class="navbar-item"><ul class="navbar-icon-links"
     aria-label="Icon Links">
         <li class="nav-item">


           <a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a>
         </li>
 </ul></div>

       </div>

   </div>

     <div class="sidebar-primary-items__start sidebar-primary__section">
         <div class="sidebar-primary-item pst-sidebar-collapse"><button id="pst-collapse-sidebar-button" aria-expanded="true" aria-controls="pst-primary-sidebar">
   <svg class="pst-icon" role="img" aria-hidden="true" focusable="false" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg">
     <path fill="currentColor" d="M3 15.5C2.36232 15.5 1.74874 15.2564 1.28478 14.8189C0.820828 14.3815 0.541576 13.7832 0.504167 13.1467L0.5 13L0.5 3C0.499965 2.36232 0.743605 1.74874 1.18107 1.28478C1.61854 0.820828 2.21676 0.541576 2.85333 0.504167L3 0.5L13 0.5C13.6377 0.499965 14.2513 0.743605 14.7152 1.18107C15.1792 1.61854 15.4584 2.21676 15.4958 2.85333L15.5 3L15.5 13C15.5 13.6377 15.2564 14.2513 14.8189 14.7152C14.3815 15.1792 13.7832 15.4584 13.1467 15.4958L13 15.5L3 15.5ZM3 13.8333L10.5 13.8333L10.5 2.16667L3 2.16667C2.79589 2.16669 2.59889 2.24163 2.44636 2.37726C2.29383 2.5129 2.19638 2.69979 2.1725 2.9025L2.16667 3L2.16667 13C2.16669 13.2041 2.24163 13.4011 2.37726 13.5536C2.5129 13.7062 2.69979 13.8036 2.9025 13.8275L3 13.8333ZM6.65583 10.325L6.5775 10.2558L4.91083 8.58917C4.76735 8.44567 4.68116 8.25476 4.66843 8.05223C4.65569 7.84971 4.71729 7.6495 4.84167 7.48917L4.91083 7.41083L6.5775 5.74417C6.72747 5.59471 6.9287 5.50794 7.14032 5.50148C7.35194 5.49502 7.55809 5.56935 7.7169 5.70937C7.8757 5.8494 7.97525 6.04463 7.99533 6.25539C8.01541 6.46616 7.95451 6.67667 7.825 6.84417L7.75583 6.9225L6.67917 8L7.75583 9.0775C7.89931 9.22099 7.98551 9.41191 7.99824 9.61443C8.01097 9.81695 7.94938 10.0172 7.825 10.1775L7.75583 10.2558C7.61234 10.3993 7.42142 10.4855 7.2189 10.4982C7.01638 10.511 6.81617 10.4494 6.65583 10.325Z"/>
   </svg>
   <span class="pst-collapse-sidebar-label">Collapse Sidebar</span>
   <span class="pst-expand-sidebar-label">Expand Sidebar</span>
 </button></div>
         <div class="sidebar-primary-item">

 <nav class="bd-docs-nav bd-links"
      aria-label="Section Navigation">
   <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
   <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav">
 <li class="toctree-l1"><a class="reference internal" href="catalog.html">Catalog</a></li>
 <li class="toctree-l1"><a class="reference internal" href="schema.html">Schema</a></li>
 <li class="toctree-l1"><a class="reference internal" href="snapshot.html">Snapshot</a></li>
 <li class="toctree-l1"><a class="reference internal" href="manifest.html">Manifest</a></li>
 <li class="toctree-l1"><a class="reference internal" href="data_types.html">Data Types</a></li>
 <li class="toctree-l1"><a class="reference internal" href="primary_key_table.html">Primary Key Table</a></li>
 <li class="toctree-l1"><a class="reference internal" href="append_only_table.html">Append Only Table</a></li>
 <li class="toctree-l1 current active"><a class="current reference internal" href="#">Write</a></li>
 <li class="toctree-l1"><a class="reference internal" href="commit.html">Commit</a></li>
 <li class="toctree-l1"><a class="reference internal" href="compaction.html">Compaction</a></li>
 <li class="toctree-l1"><a class="reference internal" href="read.html">Read</a></li>
 <li class="toctree-l1"><a class="reference internal" href="clean.html">Data Cleanup</a></li>
 <li class="toctree-l1"><a class="reference internal" href="prefetch.html">Prefetch</a></li>


 <li class="toctree-l1"><a class="reference internal" href="arrow.html">Memory Format</a></li>
 <li class="toctree-l1"><a class="reference internal" href="global_index.html">Global Index</a></li>
 </ul>
 </div>
 </nav></div>
     </div>


   <div class="sidebar-primary-items__end sidebar-primary__section">
       <div class="sidebar-primary-item">
 <div id="ethical-ad-placement"
       class="flat"
       data-ea-publisher="readthedocs"
       data-ea-type="readthedocs-sidebar"
       data-ea-manual="true">
 </div></div>
   </div>


       </div>

       <main id="main-content" class="bd-main" role="main">


           <div class="bd-content">
             <div class="bd-article-container">

               <div class="bd-header-article d-print-none">
 <div class="header-article-items header-article__inner">

     <div class="header-article-items__start">

         <div class="header-article-item">

 <nav aria-label="Breadcrumb" class="d-print-none">
   <ul class="bd-breadcrumbs">

     <li class="breadcrumb-item breadcrumb-home">
       <a href="../index.html" class="nav-link" aria-label="Home">
         <i class="fa-solid fa-home"></i>
       </a>
     </li>

     <li class="breadcrumb-item"><a href="../user_guide.html" class="nav-link">User Guide</a></li>

     <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Write</span></li>
   </ul>
 </nav>
 </div>

     </div>


 </div>
 </div>


 <div id="searchbox"></div>
                 <article class="bd-article">

   <section id="write">
 <h1>Write<a class="headerlink" href="#write" title="Link to this heading">#</a></h1>
 <p>Batch writing requires the compute engine to specify the target <code class="docutils literal notranslate"><span class="pre">partition</span></code>.
 For fixed-bucket tables, the engine must also assign a valid bucket to every
 <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>. In unaware-bucket and postpone-bucket modes, the writer can
 resolve the bucket automatically when it is omitted.</p>
 <p>Paimon C++ uses Apache Arrow as the <a class="reference internal" href="arrow.html#memory-format"><span class="std std-ref">in-memory columnar format</span></a>
 to more efficiently support writing to disk columnar formats such as ORC,
 Parquet, and Avro, thereby improving write throughput.</p>
 <div class="admonition note">
 <p class="admonition-title">Note</p>
 <dl class="simple">
 <dt>Currently supported table types:</dt><dd><ul class="simple">
 <li><p>Append table</p></li>
 <li><p>Primary Key table</p></li>
 </ul>
 </dd>
 <dt>Not supported in the current scope:</dt><dd><ul class="simple">
 <li><p>Changelog</p></li>
 </ul>
 </dd>
 </dl>
 </div>
 <section id="bucketing-modes">
 <h2>Bucketing Modes<a class="headerlink" href="#bucketing-modes" title="Link to this heading">#</a></h2>
 <ul class="simple">
 <li><p>Append tables:</p>
 <ul>
 <li><p>Support <code class="docutils literal notranslate"><span class="pre">bucket</span> <span class="pre">=</span> <span class="pre">-1</span></code> (unaware-bucket mode)</p></li>
 <li><p>Support <code class="docutils literal notranslate"><span class="pre">bucket</span> <span class="pre">&gt;</span> <span class="pre">0</span></code> (fixed bucket mode)</p></li>
 </ul>
 </li>
 <li><p>PK tables:</p>
 <ul>
 <li><p>Support <code class="docutils literal notranslate"><span class="pre">bucket</span> <span class="pre">=</span> <span class="pre">-2</span></code> (postpone bucket mode)</p></li>
 <li><p>Support <code class="docutils literal notranslate"><span class="pre">bucket</span> <span class="pre">&gt;</span> <span class="pre">0</span></code> (fixed bucket mode)</p></li>
 </ul>
 </li>
 </ul>
 <div class="admonition note">
 <p class="admonition-title">Note</p>
 <p>PK tables do not support dynamic bucketing (<code class="docutils literal notranslate"><span class="pre">bucket</span> <span class="pre">=</span> <span class="pre">-1</span></code>).</p>
 </div>
 </section>
 <section id="recordbatch-construction">
 <h2>RecordBatch Construction<a class="headerlink" href="#recordbatch-construction" title="Link to this heading">#</a></h2>
 <ul class="simple">
 <li><p>The compute engine must:</p>
 <ul>
 <li><p>Assign the correct <code class="docutils literal notranslate"><span class="pre">partition</span></code> for each row.</p></li>
 <li><p>In fixed-bucket mode, apply the Paimon-consistent bucketing function, set a
 bucket in <code class="docutils literal notranslate"><span class="pre">[0,</span> <span class="pre">bucket)</span></code>, and group rows into Arrow <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> objects
 per partition-bucket combination.</p></li>
 </ul>
 </li>
 <li><p>In unaware-bucket mode (append table with <code class="docutils literal notranslate"><span class="pre">bucket</span> <span class="pre">=</span> <span class="pre">-1</span></code>), an omitted bucket
 is resolved to <code class="docutils literal notranslate"><span class="pre">0</span></code>.</p></li>
 <li><p>In postpone-bucket mode (primary-key table with <code class="docutils literal notranslate"><span class="pre">bucket</span> <span class="pre">=</span> <span class="pre">-2</span></code>), an omitted
 bucket is resolved to <code class="docutils literal notranslate"><span class="pre">-2</span></code>.</p></li>
 <li><p>Recommended practices:</p>
 <ul>
 <li><p>Use schema-aligned Arrow arrays with explicit validity bitmaps and offsets.</p></li>
 <li><p>Prefer batch sizes tuned for I/O throughput (e.g., tens to hundreds of MB per flush, depending on filesystem and cluster configuration).</p></li>
 <li><p>Maintain stable sort orders within a batch only if required by downstream merge or compaction logic; otherwise avoid unnecessary ordering costs.</p></li>
 </ul>
 </li>
 </ul>
 </section>
 <section id="prepare-commit">
 <h2>Prepare Commit<a class="headerlink" href="#prepare-commit" title="Link to this heading">#</a></h2>
 <p>The compute engine is responsible for triggering the writer nodes’ <code class="docutils literal notranslate"><span class="pre">PrepareCommit</span></code>.
 Triggering conditions depend on the engine’s business needs and can follow either:</p>
 <ul class="simple">
 <li><p>Streaming mode: time-based or periodic triggers (e.g., every N seconds).</p></li>
 <li><p>Batch mode: trigger after all data in the batch has been written.</p></li>
 </ul>
 <p>Once the compute engine collects <code class="docutils literal notranslate"><span class="pre">CommitMessages</span></code> from all writer nodes, it
 can issue a <code class="docutils literal notranslate"><span class="pre">Commit</span></code> request to the control plane (management path) to create
 a new <code class="docutils literal notranslate"><span class="pre">Snapshot</span></code>.</p>
 <section id="compatibility-goals">
 <h3>Compatibility Goals<a class="headerlink" href="#compatibility-goals" title="Link to this heading">#</a></h3>
 <p>To ensure interoperability, the <code class="docutils literal notranslate"><span class="pre">PrepareCommit</span></code> result produced by Paimon C++
 must be consumable by Paimon Java. Therefore:</p>
 <ul class="simple">
 <li><p>The structure and semantics of <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> must remain consistent with
 Java Paimon.</p></li>
 <li><p>Any evolution of the Java-side <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> schema must be tracked and
 validated on the C++ side to maintain cross-language compatibility.</p></li>
 </ul>
 </section>
 <section id="interface-design-in-paimon-c">
 <h3>Interface Design in Paimon C++<a class="headerlink" href="#interface-design-in-paimon-c" title="Link to this heading">#</a></h3>
 <p>Unlike Java Paimon, Paimon C++ does not expose <code class="docutils literal notranslate"><span class="pre">BinaryRow</span></code>-like types in its
 public interfaces. To preserve compatibility without leaking internal row
 representations, Paimon C++ provides <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> only through:</p>
 <ul class="simple">
 <li><p>Serialization: convert the internal commit state into a well-defined binary
 representation that matches Java Paimon’s expectations.</p></li>
 <li><p>Deserialization: parse the Java-compatible binary representation back into
 C++ commit structures for validation, replay, or tooling needs.</p></li>
 </ul>
 <p>This design ensures that:</p>
 <ul class="simple">
 <li><p>Public APIs are independent of Java-specific row abstractions.</p></li>
 <li><p>Cross-language commit payloads remain stable and versionable.</p></li>
 <li><p>Internal data layouts can evolve without breaking external consumers.</p></li>
 </ul>
 </section>
 <section id="commitmessage-contract">
 <h3>CommitMessage Contract<a class="headerlink" href="#commitmessage-contract" title="Link to this heading">#</a></h3>
 <p>The <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> must encode all information required by the coordinator to
 produce a correct <code class="docutils literal notranslate"><span class="pre">Snapshot</span></code>, which commonly includes (but is not limited to):</p>
 <ul class="simple">
 <li><p>Partition and bucket identifiers associated with written data.</p></li>
 <li><p>New data files, delete files (as applicable to the table type).</p></li>
 <li><p>File-level metadata required for manifest and index updates (e.g., row counts, min/max statistics where applicable).</p></li>
 <li><p>Transactional markers and sequence numbers as required by table semantics.</p></li>
 <li><p>Any per-writer state necessary for deduplication or idempotent commits.</p></li>
 </ul>
 <div class="admonition note">
 <p class="admonition-title">Note</p>
 <p>The C++ writer supports Append and PK tables and can produce
 <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> objects for both. <code class="docutils literal notranslate"><span class="pre">FileStoreCommit</span></code> currently executes
 local commits only for append-only tables on non-object-store file systems.
 PK and object-store commit messages must be sent to an external control
 plane. Changelog is out of scope and should not be emitted in
 <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> until explicitly supported.</p>
 </div>
 </section>
 <section id="serialization-and-deserialization">
 <h3>Serialization and Deserialization<a class="headerlink" href="#serialization-and-deserialization" title="Link to this heading">#</a></h3>
 <ul class="simple">
 <li><p><strong>Binary format:</strong> The binary payload must strictly conform to Java Paimon’s
 <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> encoding. It does not contain a version tag, so callers
 must transport <code class="docutils literal notranslate"><span class="pre">CommitMessage::CurrentVersion()</span></code> separately and supply it
 when deserializing.</p></li>
 <li><p><strong>Serialization API:</strong> Use <code class="docutils literal notranslate"><span class="pre">CommitMessage::Serialize</span></code> for one message or
 <code class="docutils literal notranslate"><span class="pre">CommitMessage::SerializeList</span></code> for a list.</p></li>
 <li><p><strong>Deserialization API:</strong> Use <code class="docutils literal notranslate"><span class="pre">CommitMessage::Deserialize</span></code> or
 <code class="docutils literal notranslate"><span class="pre">CommitMessage::DeserializeList</span></code> with the separately supplied
 serialization version.</p></li>
 <li><p><strong>Validation:</strong> Conformance and round-trip tests must verify compatibility
 with Java Paimon for supported message versions.</p></li>
 </ul>
 </section>
 <section id="operational-flow">
 <h3>Operational Flow<a class="headerlink" href="#operational-flow" title="Link to this heading">#</a></h3>
 <ol class="arabic simple">
 <li><p>Writer nodes perform data ingestion and produce Arrow <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>
 organized by partition and bucket.</p></li>
 <li><p>Writers flush batches into ORC/Parquet files via registered <code class="docutils literal notranslate"><span class="pre">file.format</span></code>
 and <code class="docutils literal notranslate"><span class="pre">file-system</span></code> backends, producing file-level metadata and per-batch
 commit state.</p></li>
 <li><p>Each writer invokes <code class="docutils literal notranslate"><span class="pre">PrepareCommit</span></code>, which:
 - Aggregates per-writer state into a <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code>.
 - Returns <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> objects; it does not serialize them.</p></li>
 <li><p>The compute engine gathers <code class="docutils literal notranslate"><span class="pre">CommitMessage</span></code> objects from all writers. For
 cross-process transport, it explicitly calls <code class="docutils literal notranslate"><span class="pre">Serialize</span></code> or
 <code class="docutils literal notranslate"><span class="pre">SerializeList</span></code> and carries <code class="docutils literal notranslate"><span class="pre">CurrentVersion()</span></code> alongside the payload.</p></li>
 <li><p>For a supported local append-table commit, the engine passes the objects to
 <code class="docutils literal notranslate"><span class="pre">FileStoreCommit</span></code>. For PK tables or object-store paths, it sends the
 serialized payload and version to an external control plane.</p></li>
 <li><p>The local committer or external coordinator validates the messages, updates
 manifests/metadata, and finalizes the snapshot atomically.</p></li>
 </ol>
 </section>
 </section>
 </section>


                 </article>


                 <footer class="prev-next-footer d-print-none">

 <div class="prev-next-area">
     <a class="left-prev"
        href="append_only_table.html"
        title="previous page">
       <i class="fa-solid fa-angle-left"></i>
       <div class="prev-next-info">
         <p class="prev-next-subtitle">previous</p>
         <p class="prev-next-title">Append Only Table</p>
       </div>
     </a>
     <a class="right-next"
        href="commit.html"
        title="next page">
       <div class="prev-next-info">
         <p class="prev-next-subtitle">next</p>
         <p class="prev-next-title">Commit</p>
       </div>
       <i class="fa-solid fa-angle-right"></i>
     </a>
 </div>
                 </footer>

             </div>


                 <dialog id="pst-secondary-sidebar-modal"></dialog>
                 <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">


   <div class="sidebar-secondary-item">
 <div
     id="pst-page-navigation-heading-2"
     class="page-toc tocsection onthispage">
     <i class="fa-solid fa-list"></i> On this page
   </div>
   <nav id="pst-page-toc-nav" class="page-toc" aria-labelledby="pst-page-navigation-heading-2">
     <ul class="pst-show_toc_level nav section-nav flex-column">
 <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#bucketing-modes">Bucketing Modes</a></li>
 <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#recordbatch-construction">RecordBatch Construction</a></li>
 <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#prepare-commit">Prepare Commit</a><ul class="pst-show_toc_level nav section-nav flex-column">
 <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#compatibility-goals">Compatibility Goals</a></li>
 <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#interface-design-in-paimon-c">Interface Design in Paimon C++</a></li>
 <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#commitmessage-contract">CommitMessage Contract</a></li>
 <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#serialization-and-deserialization">Serialization and Deserialization</a></li>
 <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#operational-flow">Operational Flow</a></li>
 </ul>
 </li>
 </ul>
   </nav></div>

   <div class="sidebar-secondary-item">


   <div class="tocsection editthispage">
     <a href="https://github.com/apache/paimon-cpp/edit/main/docs/source/user_guide/write.rst">
       <i class="fa-solid fa-pencil"></i>


           Edit on GitHub


     </a>
   </div>
 </div>

 </div></div>


           </div>
           <footer class="bd-footer-content">

           </footer>

       </main>
     </div>
   </div>

   <!-- Scripts loaded after <body> so the DOM is not blocked -->
   <script defer src="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9"></script>
 <script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9"></script>

   <footer class="bd-footer">
 <div class="bd-footer__inner bd-page-width">

     <div class="footer-items__start">

         <div class="footer-item">

   <p class="copyright">

       © Copyright 2024-2026 Apache Software Foundation.
       <br/>

   </p>
 </div>

         <div class="footer-item">

   <p class="sphinx-version">
     Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 9.1.0.
     <br/>
   </p>
 </div>

     </div>


     <div class="footer-items__end">

         <div class="footer-item">
 <p class="theme-version">
   <!-- # L10n: Setting the PST URL as an argument as this does not need to be localized -->
   Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.20.0.
 </p></div>

     </div>

 </div>

   </footer>
   </body>
 </html>