| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <title>Manifest — C++ Paimon</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || ""; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=90905a2f556bf617f1a9" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" /> |
| <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=d7227091" /> |
| |
| <!-- So that users can add custom icons --> |
| <script defer src="../_static/scripts/fontawesome.js?digest=90905a2f556bf617f1a9"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9" /> |
| |
| <script src="../_static/documentation_options.js?v=5929fcd5"></script> |
| <script src="../_static/doctools.js?v=fd6eb6e6"></script> |
| <script src="../_static/sphinx_highlight.js?v=6ffebe34"></script> |
| <script src="../_static/clipboard.min.js?v=a7894cd8"></script> |
| <script src="../_static/copybutton.js?v=3bb21c8c"></script> |
| <script src="../_static/design-tabs.js?v=f930bc37"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'user_guide/manifest';</script> |
| <script>DOCUMENTATION_OPTIONS.search_as_you_type = false;</script> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Data Types" href="data_types.html" /> |
| <link rel="prev" title="Snapshot" href="snapshot.html" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="0.2.3" /> |
| |
| |
| <script src="../_static/searchtools.js"></script> |
| <script src="../_static/language_data.js"></script> |
| <script src="../searchindex.js"></script> |
| |
| </head> |
| <body data-default-mode=""> |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header id="pst-header" class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class=" navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| <p class="title logo__title">Paimon C++</p> |
| |
| </a></div> |
| |
| </div> |
| |
| <div class=" navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../getting_started.html"> |
| Getting Started |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../user_guide.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../examples/index.html"> |
| Examples |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../api.html"> |
| API Reference |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| |
| <div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode"> |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown"> |
| <i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i> |
| </button> |
| <ul class="dropdown-menu dropdown-menu-end"> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li> |
| </ul> |
| </div></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> |
| <span class="fa-solid fa-outdent"></span> |
| </button> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../getting_started.html"> |
| Getting Started |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="../user_guide.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../examples/index.html"> |
| Examples |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../api.html"> |
| API Reference |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| |
| <div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode"> |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown"> |
| <i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i> |
| </button> |
| <ul class="dropdown-menu dropdown-menu-end"> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li> |
| </ul> |
| </div></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item pst-sidebar-collapse"><button id="pst-collapse-sidebar-button" aria-expanded="true" aria-controls="pst-primary-sidebar"> |
| <svg class="pst-icon" role="img" aria-hidden="true" focusable="false" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"> |
| <path fill="currentColor" d="M3 15.5C2.36232 15.5 1.74874 15.2564 1.28478 14.8189C0.820828 14.3815 0.541576 13.7832 0.504167 13.1467L0.5 13L0.5 3C0.499965 2.36232 0.743605 1.74874 1.18107 1.28478C1.61854 0.820828 2.21676 0.541576 2.85333 0.504167L3 0.5L13 0.5C13.6377 0.499965 14.2513 0.743605 14.7152 1.18107C15.1792 1.61854 15.4584 2.21676 15.4958 2.85333L15.5 3L15.5 13C15.5 13.6377 15.2564 14.2513 14.8189 14.7152C14.3815 15.1792 13.7832 15.4584 13.1467 15.4958L13 15.5L3 15.5ZM3 13.8333L10.5 13.8333L10.5 2.16667L3 2.16667C2.79589 2.16669 2.59889 2.24163 2.44636 2.37726C2.29383 2.5129 2.19638 2.69979 2.1725 2.9025L2.16667 3L2.16667 13C2.16669 13.2041 2.24163 13.4011 2.37726 13.5536C2.5129 13.7062 2.69979 13.8036 2.9025 13.8275L3 13.8333ZM6.65583 10.325L6.5775 10.2558L4.91083 8.58917C4.76735 8.44567 4.68116 8.25476 4.66843 8.05223C4.65569 7.84971 4.71729 7.6495 4.84167 7.48917L4.91083 7.41083L6.5775 5.74417C6.72747 5.59471 6.9287 5.50794 7.14032 5.50148C7.35194 5.49502 7.55809 5.56935 7.7169 5.70937C7.8757 5.8494 7.97525 6.04463 7.99533 6.25539C8.01541 6.46616 7.95451 6.67667 7.825 6.84417L7.75583 6.9225L6.67917 8L7.75583 9.0775C7.89931 9.22099 7.98551 9.41191 7.99824 9.61443C8.01097 9.81695 7.94938 10.0172 7.825 10.1775L7.75583 10.2558C7.61234 10.3993 7.42142 10.4855 7.2189 10.4982C7.01638 10.511 6.81617 10.4494 6.65583 10.325Z"/> |
| </svg> |
| <span class="pst-collapse-sidebar-label">Collapse Sidebar</span> |
| <span class="pst-expand-sidebar-label">Expand Sidebar</span> |
| </button></div> |
| <div class="sidebar-primary-item"> |
| |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="catalog.html">Catalog</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="schema.html">Schema</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="snapshot.html">Snapshot</a></li> |
| <li class="toctree-l1 current active"><a class="current reference internal" href="#">Manifest</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="data_types.html">Data Types</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="primary_key_table.html">Primary Key Table</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="append_only_table.html">Append Only Table</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="write.html">Write</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="commit.html">Commit</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="compaction.html">Compaction</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="read.html">Read</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="clean.html">Data Cleanup</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="prefetch.html">Prefetch</a></li> |
| |
| |
| |
| <li class="toctree-l1"><a class="reference internal" href="arrow.html">Memory Format</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="global_index.html">Global Index</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../user_guide.html" class="nav-link">User Guide</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Manifest</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="manifest"> |
| <h1>Manifest<a class="headerlink" href="#manifest" title="Link to this heading">#</a></h1> |
| <section id="manifest-list"> |
| <h2>Manifest List<a class="headerlink" href="#manifest-list" title="Link to this heading">#</a></h2> |
| <div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>├──<span class="w"> </span>manifest |
| <span class="w"> </span>└──<span class="w"> </span>manifest-list-51c16f7b-421c-4bc0-80a0-17677f343358-1 |
| </pre></div> |
| </div> |
| <p>Manifest List includes metadata of several manifest files. Its name contains a UUID. It is an Avro file with the following schema:</p> |
| <ol class="arabic simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE_NAME</span></code>: STRING — manifest file name.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE_SIZE</span></code>: BIGINT — manifest file size.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_NUM_ADDED_FILES</span></code>: BIGINT — number of added files in the manifest.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_NUM_DELETED_FILES</span></code>: BIGINT — number of deleted files in the manifest.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_PARTITION_STATS</span></code>: SimpleStats — partition stats. The minimum and maximum values of partition fields in this manifest are beneficial for skipping certain manifest files during queries.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_SCHEMA_ID</span></code>: BIGINT — schema id used when writing this manifest file.</p></li> |
| </ol> |
| </section> |
| <section id="id1"> |
| <h2>Manifest<a class="headerlink" href="#id1" title="Link to this heading">#</a></h2> |
| <p>Manifest includes metadata of several data files, changelog files, or table-index files. Its name contains a UUID, and it is an Avro file.</p> |
| <p>The changes of the file are saved in the manifest, and a file can be added or deleted. Manifests should be in an orderly manner, and the same file may be added or deleted multiple times. The last version should be read. This design makes commit lighter to support file deletion generated by compaction.</p> |
| </section> |
| <section id="data-manifest"> |
| <h2>Data Manifest<a class="headerlink" href="#data-manifest" title="Link to this heading">#</a></h2> |
| <p>Data Manifest includes metadata of several data files or changelog files.</p> |
| <div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>├──<span class="w"> </span>manifest |
| <span class="w"> </span>└──<span class="w"> </span>manifest-6758823b-2010-4d06-aef0-3b1b597723d6-0 |
| </pre></div> |
| </div> |
| <p>Schema:</p> |
| <ol class="arabic simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_KIND</span></code>: TINYINT — <code class="docutils literal notranslate"><span class="pre">ADD</span></code> or <code class="docutils literal notranslate"><span class="pre">DELETE</span></code>.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_PARTITION</span></code>: BYTES — partition spec, a BinaryRow.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_BUCKET</span></code>: INT — bucket of this file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_TOTAL_BUCKETS</span></code>: INT — total buckets when writing this file; used for verification after bucket changes.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE</span></code>: data file metadata.</p></li> |
| </ol> |
| <p>Data file metadata:</p> |
| <ol class="arabic simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE_NAME</span></code>: STRING — file name.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE_SIZE</span></code>: BIGINT — file size.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_ROW_COUNT</span></code>: BIGINT — total number of rows (including add & delete) in this file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_MIN_KEY</span></code>: STRING — minimum key of this file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_MAX_KEY</span></code>: STRING — maximum key of this file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_KEY_STATS</span></code>: SimpleStats — statistics of the key.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_VALUE_STATS</span></code>: SimpleStats — statistics of the value.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_MIN_SEQUENCE_NUMBER</span></code>: BIGINT — minimum sequence number.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_MAX_SEQUENCE_NUMBER</span></code>: BIGINT — maximum sequence number.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_SCHEMA_ID</span></code>: BIGINT — schema id when writing this file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_LEVEL</span></code>: INT — level of this file in LSM.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_EXTRA_FILES</span></code>: ARRAY<STRING> — extra files for this file (e.g., data file index file).</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_CREATION_TIME</span></code>: TIMESTAMP_MILLIS — creation time of this file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_DELETE_ROW_COUNT</span></code>: BIGINT — rowCount = addRowCount + deleteRowCount.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_EMBEDDED_FILE_INDEX</span></code>: BYTES — if the data file index is small, store the index in the manifest.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE_SOURCE</span></code>: TINYINT — indicates whether this file is generated as an <code class="docutils literal notranslate"><span class="pre">APPEND</span></code> or <code class="docutils literal notranslate"><span class="pre">COMPACT</span></code> file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_VALUE_STATS_COLS</span></code>: ARRAY<STRING> — statistical columns in metadata.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_EXTERNAL_PATH</span></code>: STRING — external path of this file; <code class="docutils literal notranslate"><span class="pre">null</span></code> if it is in the warehouse.</p></li> |
| </ol> |
| </section> |
| <section id="index-manifest"> |
| <h2>Index Manifest<a class="headerlink" href="#index-manifest" title="Link to this heading">#</a></h2> |
| <p>Index Manifest includes metadata of several table-index files.</p> |
| <div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>├──<span class="w"> </span>manifest |
| <span class="w"> </span>└──<span class="w"> </span>index-manifest-5d670043-da25-4265-9a26-e31affc98039-0 |
| </pre></div> |
| </div> |
| <p>Schema:</p> |
| <ol class="arabic simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_KIND</span></code>: TINYINT — <code class="docutils literal notranslate"><span class="pre">ADD</span></code> or <code class="docutils literal notranslate"><span class="pre">DELETE</span></code>.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_PARTITION</span></code>: BYTES — partition spec, a BinaryRow.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_BUCKET</span></code>: INT — bucket of this file.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_INDEX_TYPE</span></code>: STRING — <code class="docutils literal notranslate"><span class="pre">HASH</span></code> or <code class="docutils literal notranslate"><span class="pre">DELETION_VECTORS</span></code>.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE_NAME</span></code>: STRING — file name.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_FILE_SIZE</span></code>: BIGINT — file size.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_ROW_COUNT</span></code>: BIGINT — total number of rows.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_DELETIONS_VECTORS_RANGES</span></code>: Metadata only used by <code class="docutils literal notranslate"><span class="pre">DELETION_VECTORS</span></code>; an array of deletion vector metadata. Each deletion vector metadata has: |
| - <code class="docutils literal notranslate"><span class="pre">f0</span></code>: the data file name corresponding to this deletion vector. |
| - <code class="docutils literal notranslate"><span class="pre">f1</span></code>: the starting offset of this deletion vector in the index file. |
| - <code class="docutils literal notranslate"><span class="pre">f2</span></code>: the length of this deletion vector in the index file. |
| - <code class="docutils literal notranslate"><span class="pre">_CARDINALITY</span></code>: the number of deleted rows.</p></li> |
| </ol> |
| </section> |
| <section id="appendix"> |
| <h2>Appendix<a class="headerlink" href="#appendix" title="Link to this heading">#</a></h2> |
| <section id="simplestats"> |
| <h3>SimpleStats<a class="headerlink" href="#simplestats" title="Link to this heading">#</a></h3> |
| <p>SimpleStats is a nested row with the following schema:</p> |
| <ol class="arabic simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_MIN_VALUES</span></code>: BYTES — BinaryRow; the minimum values of the columns.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_MAX_VALUES</span></code>: BYTES — BinaryRow; the maximum values of the columns.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">_NULL_COUNTS</span></code>: ARRAY<BIGINT> — the number of nulls in the columns.</p></li> |
| </ol> |
| </section> |
| <section id="binaryrow"> |
| <h3>BinaryRow<a class="headerlink" href="#binaryrow" title="Link to this heading">#</a></h3> |
| <p>BinaryRow is backed by bytes instead of <code class="docutils literal notranslate"><span class="pre">Object</span></code>. It can significantly reduce the serialization/deserialization of Java objects.</p> |
| <p>A row has two parts: fixed-length part and variable-length part.</p> |
| <ul class="simple"> |
| <li><p>Fixed-length part contains a 1-byte header, null bit set, and field values. The null bit set is used for null tracking and is aligned to 8-byte word boundaries.</p></li> |
| <li><p>Field values hold fixed-length primitive types and variable-length values that can be stored in 8 bytes. If the variable-length field does not fit in 8 bytes, then the fixed-length part stores the length and the offset of the variable-length part.</p></li> |
| </ul> |
| </section> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="snapshot.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Snapshot</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="data_types.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Data Types</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <dialog id="pst-secondary-sidebar-modal"></dialog> |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav id="pst-page-toc-nav" class="page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="pst-show_toc_level nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#manifest-list">Manifest List</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id1">Manifest</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#data-manifest">Data Manifest</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#index-manifest">Index Manifest</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#appendix">Appendix</a><ul class="pst-show_toc_level nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#simplestats">SimpleStats</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#binaryrow">BinaryRow</a></li> |
| </ul> |
| </li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/paimon-cpp/edit/main/docs/source/user_guide/manifest.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../_static/scripts/bootstrap.js?digest=90905a2f556bf617f1a9"></script> |
| <script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=90905a2f556bf617f1a9"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2024-2026 Apache Software Foundation. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 9.1.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| <!-- # L10n: Setting the PST URL as an argument as this does not need to be localized --> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.20.0. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
