| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="../" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <title>Benchmark Usage — Apache Paimon C++</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || ""; |
| </script> |
| <!-- |
| this give us a css class that will be invisible only if js is disabled |
| --> |
| <noscript> |
| <style> |
| .pst-js-only { display: none !important; } |
| |
| </style> |
| </noscript> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=cb8930eaaf36d9849b67" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=cb8930eaaf36d9849b67" rel="stylesheet" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" /> |
| <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css?v=d7227091" /> |
| |
| <!-- So that users can add custom icons --> |
| <script defer src="../_static/scripts/fontawesome.js?digest=cb8930eaaf36d9849b67"></script> |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=cb8930eaaf36d9849b67" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=cb8930eaaf36d9849b67" /> |
| |
| <script src="../_static/documentation_options.js?v=5929fcd5"></script> |
| <script src="../_static/doctools.js?v=fd6eb6e6"></script> |
| <script src="../_static/sphinx_highlight.js?v=6ffebe34"></script> |
| <script src="../_static/clipboard.min.js?v=a7894cd8"></script> |
| <script src="../_static/copybutton.js?v=3bb21c8c"></script> |
| <script src="../_static/design-tabs.js?v=f930bc37"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'examples/benchmark';</script> |
| <script>DOCUMENTATION_OPTIONS.search_as_you_type = false;</script> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="API Reference" href="../api.html" /> |
| <link rel="prev" title="Clean Example" href="clean.html" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| <meta name="docsearch:version" content="0.4.0" /> |
| |
| |
| <script src="../_static/searchtools.js"></script> |
| <script src="../_static/language_data.js"></script> |
| <script src="../searchindex.js"></script> |
| |
| </head> |
| <body data-default-mode=""> |
| |
| |
| <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div> |
| |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i>Back to top</button> |
| |
| |
| |
| |
| <dialog id="pst-search-dialog"> |
| |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form> |
| </dialog> |
| |
| <div class="pst-async-banner-revealer d-none"> |
| <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside> |
| </div> |
| |
| |
| <header id="pst-header" class="bd-header navbar navbar-expand-lg bd-navbar d-print-none"> |
| <div class="bd-header__inner bd-page-width"> |
| <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation"> |
| <span class="fa-solid fa-bars"></span> |
| </button> |
| |
| |
| <div class=" navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| <p class="title logo__title">Apache Paimon C++</p> |
| |
| </a></div> |
| |
| </div> |
| |
| <div class=" navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../getting_started.html"> |
| Getting Started |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../user_guide.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Examples |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../api.html"> |
| API Reference |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| |
| <div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode"> |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown"> |
| <i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i> |
| </button> |
| <ul class="dropdown-menu dropdown-menu-end"> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li> |
| </ul> |
| </div></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| </div> |
| |
| |
| |
| <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page"> |
| <span class="fa-solid fa-outdent"></span> |
| </button> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <dialog id="pst-primary-sidebar-modal"></dialog> |
| <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| |
| |
| <div class="navbar-item"> |
| <nav> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../getting_started.html"> |
| Getting Started |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../user_guide.html"> |
| User Guide |
| </a> |
| </li> |
| |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Examples |
| </a> |
| </li> |
| |
| |
| <li class="nav-item "> |
| <a class="nav-link nav-internal" href="../api.html"> |
| API Reference |
| </a> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| |
| <div class="theme-switch-container dropdown pst-js-only" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Color mode"> |
| <button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button dropdown-toggle" aria-label="Color mode" data-bs-toggle="dropdown"> |
| <i class="theme-switch fa-solid fa-sun fa-lg fa-fw" data-mode="light" title="Light"></i> |
| <i class="theme-switch fa-solid fa-moon fa-lg fa-fw" data-mode="dark" title="Dark"></i> |
| <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg fa-fw" data-mode="auto" title="System Settings"></i> |
| </button> |
| <ul class="dropdown-menu dropdown-menu-end"> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg fa-fw me-1"></i>System Settings</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="light"><i class="fa-solid fa-sun fa-lg fa-fw me-1"></i>Light</button></li> |
| <li><button class="dropdown-item d-flex align-items-center theme-change-button" data-mode="dark"><i class="fa-solid fa-moon fa-lg fa-fw me-1"></i>Dark</button></li> |
| </ul> |
| </div></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/paimon-cpp" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i><span class="visually-hidden">GitHub</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item pst-sidebar-collapse"><button id="pst-collapse-sidebar-button" aria-expanded="true" aria-controls="pst-primary-sidebar"> |
| <svg class="pst-icon" role="img" aria-hidden="true" focusable="false" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"> |
| <path fill="currentColor" d="M3 15.5C2.36232 15.5 1.74874 15.2564 1.28478 14.8189C0.820828 14.3815 0.541576 13.7832 0.504167 13.1467L0.5 13L0.5 3C0.499965 2.36232 0.743605 1.74874 1.18107 1.28478C1.61854 0.820828 2.21676 0.541576 2.85333 0.504167L3 0.5L13 0.5C13.6377 0.499965 14.2513 0.743605 14.7152 1.18107C15.1792 1.61854 15.4584 2.21676 15.4958 2.85333L15.5 3L15.5 13C15.5 13.6377 15.2564 14.2513 14.8189 14.7152C14.3815 15.1792 13.7832 15.4584 13.1467 15.4958L13 15.5L3 15.5ZM3 13.8333L10.5 13.8333L10.5 2.16667L3 2.16667C2.79589 2.16669 2.59889 2.24163 2.44636 2.37726C2.29383 2.5129 2.19638 2.69979 2.1725 2.9025L2.16667 3L2.16667 13C2.16669 13.2041 2.24163 13.4011 2.37726 13.5536C2.5129 13.7062 2.69979 13.8036 2.9025 13.8275L3 13.8333ZM6.65583 10.325L6.5775 10.2558L4.91083 8.58917C4.76735 8.44567 4.68116 8.25476 4.66843 8.05223C4.65569 7.84971 4.71729 7.6495 4.84167 7.48917L4.91083 7.41083L6.5775 5.74417C6.72747 5.59471 6.9287 5.50794 7.14032 5.50148C7.35194 5.49502 7.55809 5.56935 7.7169 5.70937C7.8757 5.8494 7.97525 6.04463 7.99533 6.25539C8.01541 6.46616 7.95451 6.67667 7.825 6.84417L7.75583 6.9225L6.67917 8L7.75583 9.0775C7.89931 9.22099 7.98551 9.41191 7.99824 9.61443C8.01097 9.81695 7.94938 10.0172 7.825 10.1775L7.75583 10.2558C7.61234 10.3993 7.42142 10.4855 7.2189 10.4982C7.01638 10.511 6.81617 10.4494 6.65583 10.325Z"/> |
| </svg> |
| <span class="pst-collapse-sidebar-label">Collapse Sidebar</span> |
| <span class="pst-expand-sidebar-label">Expand Sidebar</span> |
| </button></div> |
| <div class="sidebar-primary-item"> |
| |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="write_commit_scan_read.html">Write Commit Scan Read Example</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="clean.html">Clean Example</a></li> |
| <li class="toctree-l1 current active"><a class="current reference internal" href="#">Benchmark Usage</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <div id="ethical-ad-placement" |
| class="flat" |
| data-ea-publisher="readthedocs" |
| data-ea-type="readthedocs-sidebar" |
| data-ea-manual="true"> |
| </div></div> |
| </div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main" role="main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article d-print-none"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| <nav aria-label="Breadcrumb" class="d-print-none"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Examples</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Benchmark Usage</span></li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="benchmark-usage"> |
| <h1>Benchmark Usage<a class="headerlink" href="#benchmark-usage" title="Link to this heading">#</a></h1> |
| <p>Paimon C++ provides Google Benchmark based cases at two levels:</p> |
| <dl class="simple"> |
| <dt><code class="docutils literal notranslate"><span class="pre">paimon-read-write-benchmark</span></code></dt><dd><p>Table-level cases for append-table write/read and primary-key table write/MOR |
| read paths.</p> |
| </dd> |
| <dt><code class="docutils literal notranslate"><span class="pre">paimon-parquet-format-benchmark</span></code></dt><dd><p>Format-level cases that drive the Parquet writer and reader directly, without |
| catalog lookup, split planning, merge/sort or commit.</p> |
| </dd> |
| </dl> |
| <p>Benchmarks are disabled by default.</p> |
| <section id="build"> |
| <h2>Build<a class="headerlink" href="#build" title="Link to this heading">#</a></h2> |
| <p>Enable benchmarks when configuring CMake:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">cmake</span> <span class="o">-</span><span class="n">S</span> <span class="o">.</span> <span class="o">-</span><span class="n">B</span> <span class="n">build</span> <span class="o">-</span><span class="n">DPAIMON_BUILD_BENCHMARKS</span><span class="o">=</span><span class="n">ON</span> |
| <span class="n">cmake</span> <span class="o">--</span><span class="n">build</span> <span class="n">build</span> <span class="o">--</span><span class="n">target</span> <span class="n">paimon</span><span class="o">-</span><span class="n">read</span><span class="o">-</span><span class="n">write</span><span class="o">-</span><span class="n">benchmark</span> |
| <span class="n">cmake</span> <span class="o">--</span><span class="n">build</span> <span class="n">build</span> <span class="o">--</span><span class="n">target</span> <span class="n">paimon</span><span class="o">-</span><span class="n">parquet</span><span class="o">-</span><span class="nb">format</span><span class="o">-</span><span class="n">benchmark</span> |
| </pre></div> |
| </div> |
| <p>Run all benchmark cases through CTest:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">cmake</span> <span class="o">--</span><span class="n">build</span> <span class="n">build</span> <span class="o">--</span><span class="n">target</span> <span class="n">benchmark</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="table-level-custom-options"> |
| <h2>Table-level Custom Options<a class="headerlink" href="#table-level-custom-options" title="Link to this heading">#</a></h2> |
| <p><code class="docutils literal notranslate"><span class="pre">paimon-read-write-benchmark</span></code> accepts Google Benchmark options plus the Paimon |
| specific options below:</p> |
| <dl class="simple"> |
| <dt><code class="docutils literal notranslate"><span class="pre">--paimon_source_data_file=<path></span></code></dt><dd><p>Source data file used to build benchmark data. Currently Parquet source files |
| are supported.</p> |
| </dd> |
| <dt><code class="docutils literal notranslate"><span class="pre">--paimon_source_table_path=<path></span></code></dt><dd><p>Read directly from an existing table path for <code class="docutils literal notranslate"><span class="pre">BM_Read</span></code> and <code class="docutils literal notranslate"><span class="pre">BM_MOR_Read</span></code>. |
| When set, the source loading and pre-write stage are skipped.</p> |
| </dd> |
| <dt><code class="docutils literal notranslate"><span class="pre">--paimon_pk_columns=<col1,col2,...></span></code></dt><dd><p>Primary key columns for <code class="docutils literal notranslate"><span class="pre">BM_PK_Write</span></code> and <code class="docutils literal notranslate"><span class="pre">BM_MOR_Read</span></code>. These cases |
| explicitly use <code class="docutils literal notranslate"><span class="pre">bucket=1</span></code> because benchmark batches are written to bucket 0.</p> |
| </dd> |
| <dt><code class="docutils literal notranslate"><span class="pre">--paimon_option=<key1>:<value1>;<key2>:<value2></span></code></dt><dd><p>Repeatable table options passed through to Paimon. The default table file |
| format is <code class="docutils literal notranslate"><span class="pre">parquet</span></code>. Benchmark output supports <code class="docutils literal notranslate"><span class="pre">parquet</span></code> and, when |
| built with <code class="docutils literal notranslate"><span class="pre">PAIMON_ENABLE_ORC=ON</span></code>, <code class="docutils literal notranslate"><span class="pre">orc</span></code>; use |
| <code class="docutils literal notranslate"><span class="pre">--paimon_option</span> <span class="pre">file.format:parquet</span></code> or |
| <code class="docutils literal notranslate"><span class="pre">--paimon_option</span> <span class="pre">file.format:orc</span></code> to select one. For <code class="docutils literal notranslate"><span class="pre">BM_PK_Write</span></code> and |
| <code class="docutils literal notranslate"><span class="pre">BM_MOR_Read</span></code>, <code class="docutils literal notranslate"><span class="pre">bucket</span></code> is forced to <code class="docutils literal notranslate"><span class="pre">1</span></code>.</p> |
| </dd> |
| </dl> |
| </section> |
| <section id="examples"> |
| <h2>Examples<a class="headerlink" href="#examples" title="Link to this heading">#</a></h2> |
| <p>Append table write:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">read</span><span class="o">-</span><span class="n">write</span><span class="o">-</span><span class="n">benchmark</span> \ |
| <span class="o">--</span><span class="n">paimon_source_data_file</span> <span class="o">/</span><span class="n">path</span><span class="o">/</span><span class="n">data</span><span class="o">.</span><span class="n">parquet</span> \ |
| <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="n">BM_Write</span> |
| </pre></div> |
| </div> |
| <p>Append table read with four prefetch workers:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">read</span><span class="o">-</span><span class="n">write</span><span class="o">-</span><span class="n">benchmark</span> \ |
| <span class="o">--</span><span class="n">paimon_source_data_file</span> <span class="o">/</span><span class="n">path</span><span class="o">/</span><span class="n">data</span><span class="o">.</span><span class="n">parquet</span> \ |
| <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="n">BM_Read</span><span class="o">/</span><span class="mi">4</span> |
| </pre></div> |
| </div> |
| <p>Primary-key table write:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">read</span><span class="o">-</span><span class="n">write</span><span class="o">-</span><span class="n">benchmark</span> \ |
| <span class="o">--</span><span class="n">paimon_source_data_file</span> <span class="o">/</span><span class="n">path</span><span class="o">/</span><span class="n">data</span><span class="o">.</span><span class="n">parquet</span> \ |
| <span class="o">--</span><span class="n">paimon_pk_columns</span><span class="o">=</span><span class="nb">id</span> \ |
| <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="n">BM_PK_Write</span> |
| </pre></div> |
| </div> |
| <p>MOR read from an existing table:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">read</span><span class="o">-</span><span class="n">write</span><span class="o">-</span><span class="n">benchmark</span> \ |
| <span class="o">--</span><span class="n">paimon_source_table_path</span> <span class="o">/</span><span class="n">path</span><span class="o">/</span><span class="n">table</span> \ |
| <span class="o">--</span><span class="n">paimon_pk_columns</span><span class="o">=</span><span class="nb">id</span> \ |
| <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="n">BM_MOR_Read</span><span class="o">/</span><span class="mi">4</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="parquet-format-benchmark"> |
| <h2>Parquet Format Benchmark<a class="headerlink" href="#parquet-format-benchmark" title="Link to this heading">#</a></h2> |
| <p><code class="docutils literal notranslate"><span class="pre">paimon-parquet-format-benchmark</span></code> takes only Google Benchmark options. It |
| generates its own data and writes it to a temporary directory, so it needs no |
| source file or table.</p> |
| <p>Two things shape how the results should be read:</p> |
| <ul class="simple"> |
| <li><p>Every axis is swept on its own rather than as a combined matrix, so each |
| case answers one question and a change can be attributed to it.</p></li> |
| <li><p>Writes go through the local FileSystem into a temporary directory, so |
| absolute numbers carry the cost of that path. Comparisons are meaningful |
| only under the same environment and methodology - the same machine, build |
| configuration and options - which is what makes a before/after comparison |
| useful.</p></li> |
| </ul> |
| <p>Writer cases (<code class="docutils literal notranslate"><span class="pre">BM_ParquetWrite_*</span></code>) cover flat primitives, <code class="docutils literal notranslate"><span class="pre">VARCHAR</span></code> at low / |
| medium / high cardinality with and without file-level dictionary encoding, |
| already dictionary-encoded <code class="docutils literal notranslate"><span class="pre">VARCHAR</span></code> / <code class="docutils literal notranslate"><span class="pre">INTEGER</span></code> input arrays against their |
| flat equivalents, <code class="docutils literal notranslate"><span class="pre">DECIMAL</span></code> at precision 9 / 18 / 38, nested <code class="docutils literal notranslate"><span class="pre">STRUCT</span></code> / |
| <code class="docutils literal notranslate"><span class="pre">LIST</span></code> / <code class="docutils literal notranslate"><span class="pre">VECTOR</span></code> / <code class="docutils literal notranslate"><span class="pre">MAP</span></code>, null density from 0 to 100 percent, rows per |
| <code class="docutils literal notranslate"><span class="pre">AddBatch</span></code> call, column count at a fixed row count, row group size, the |
| writer memory threshold that triggers a byte-based row-group flush, and the |
| codecs Parquet accepts - <code class="docutils literal notranslate"><span class="pre">none</span></code>, <code class="docutils literal notranslate"><span class="pre">snappy</span></code>, <code class="docutils literal notranslate"><span class="pre">gzip</span></code>, <code class="docutils literal notranslate"><span class="pre">brotli</span></code>, <code class="docutils literal notranslate"><span class="pre">zstd</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">lz4_raw</span></code> and <code class="docutils literal notranslate"><span class="pre">lz4_hadoop</span></code>. Note that <code class="docutils literal notranslate"><span class="pre">lz4</span></code> is deliberately not among |
| them: it resolves to Arrow’s <code class="docutils literal notranslate"><span class="pre">LZ4_FRAME</span></code>, which |
| <code class="docutils literal notranslate"><span class="pre">parquet::IsCodecSupported</span></code> rejects.</p> |
| <p>The two dictionary axes are different questions. <code class="docutils literal notranslate"><span class="pre">BM_ParquetWrite_String</span></code> and |
| <code class="docutils literal notranslate"><span class="pre">BM_ParquetWrite_StringNoDictionary</span></code> vary whether the <em>file</em> is dictionary |
| encoded; <code class="docutils literal notranslate"><span class="pre">BM_ParquetWrite_Dictionary*</span></code> vary whether the <em>input array</em> already |
| is, which is what decides whether Arrow can pass indices through to Parquet or |
| has to materialize them first.</p> |
| <p>Three of those cases form one comparison, at the same cardinality and over the |
| same logical column: <code class="docutils literal notranslate"><span class="pre">BM_ParquetWrite_String</span></code> writes it flat, |
| <code class="docutils literal notranslate"><span class="pre">BM_ParquetWrite_DictionaryStringIntoStringSchema</span></code> writes it as one dictionary |
| forwarded through the whole file, and |
| <code class="docutils literal notranslate"><span class="pre">BM_ParquetWrite_ChangingDictionaryStringIntoStringSchema</span></code> gives every batch |
| its own dictionary - a rotation of the same values, with the indices shifted the |
| other way, so the data is unchanged and only the dictionary object differs. A |
| Parquet column chunk holds one dictionary, so the third case makes the writer |
| fall back to plain encoding partway through the row group. Reading the second |
| against the first is what forwarding an encoding buys; the third against the |
| second is what that fallback costs in time; the third against the first is the |
| output size a rewrite that materialized and rebuilt would have produced.</p> |
| <p>These are format-writer microbenchmarks: they measure <code class="docutils literal notranslate"><span class="pre">AddBatch</span></code> against a |
| Parquet file, not a compaction. The compaction time, CPU and peak memory of |
| <code class="docutils literal notranslate"><span class="pre">parquet.read.enable-dictionary-passthrough</span></code> on a real table have to be |
| measured on that table - see the “Dictionary Passthrough” section of |
| <a class="reference internal" href="../user_guide/compaction.html"><span class="doc">Compaction</span></a>.</p> |
| <p>Reader cases (<code class="docutils literal notranslate"><span class="pre">BM_ParquetRead_*</span></code>) cover full scan, single-column projection, |
| predicate-filtered reads at varying selectivity with page-index filtering on and |
| off, skip-heavy reads driven by a strided selection bitmap, null density, |
| <code class="docutils literal notranslate"><span class="pre">DECIMAL</span></code> at precision 9 / 18 / 38, <code class="docutils literal notranslate"><span class="pre">DOUBLE</span></code>, dictionary-encoded against |
| plain-encoded files, rows per <code class="docutils literal notranslate"><span class="pre">NextBatch</span></code> call, and nested column reads.</p> |
| <p>Every case reports <code class="docutils literal notranslate"><span class="pre">ns_per_row</span></code> next to <code class="docutils literal notranslate"><span class="pre">bytes_per_row</span></code> - <code class="docutils literal notranslate"><span class="pre">file_bytes</span></code> for |
| writes, <code class="docutils literal notranslate"><span class="pre">read_bytes</span></code> for reads - so a change that trades CPU for size is |
| visible in both directions. Read cases additionally report <code class="docutils literal notranslate"><span class="pre">rows_read</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">batches</span></code>, and <code class="docutils literal notranslate"><span class="pre">row_groups</span></code> / <code class="docutils literal notranslate"><span class="pre">row_groups_after_filter</span></code> from the reader’s |
| own metrics.</p> |
| <p>Compare filtered cases on <code class="docutils literal notranslate"><span class="pre">ns_per_input_row</span></code> and <code class="docutils literal notranslate"><span class="pre">bytes_per_input_row</span></code>, not |
| <code class="docutils literal notranslate"><span class="pre">ns_per_row</span></code> and <code class="docutils literal notranslate"><span class="pre">bytes_per_row</span></code>. The latter pair divides by the rows a case |
| actually materialized, so pruning shrinks numerator and denominator together and |
| they can rise even as the run gets faster; the <code class="docutils literal notranslate"><span class="pre">_input_row</span></code> pair divides by the |
| rows the file holds, which every setting shares.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">row_groups_after_filter</span></code> counts row groups only. It does not show page-level |
| pruning: on the ordered <code class="docutils literal notranslate"><span class="pre">id</span></code> column both page-index settings usually keep the |
| same row groups, and the page-index gain shows up in <code class="docutils literal notranslate"><span class="pre">rows_read</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">read_bytes</span></code> and <code class="docutils literal notranslate"><span class="pre">ns_per_input_row</span></code> instead.</p> |
| <p>A case that cannot run - an unsupported codec, a schema the reader rejects - |
| calls <code class="docutils literal notranslate"><span class="pre">SkipWithError</span></code> and makes the process exit non-zero, so <code class="docutils literal notranslate"><span class="pre">ctest</span> <span class="pre">-L</span> |
| <span class="pre">benchmark</span></code> fails instead of reporting a silent skip. Read cases also assert on |
| the number of rows they materialized, so a fixture that stopped producing rows |
| fails rather than looking fast.</p> |
| <p>Because the benchmark is only compiled under <code class="docutils literal notranslate"><span class="pre">PAIMON_BUILD_BENCHMARKS</span></code>, the |
| format-layer assumptions it relies on are covered separately by |
| <code class="docutils literal notranslate"><span class="pre">paimon-parquet-format-benchmark-test</span></code>, which builds with the normal test |
| suite.</p> |
| <p>Each read case scans a file that is generated once on first use and reused for |
| the rest of the run, so a filtered run only pays to build the fixtures its own |
| cases need.</p> |
| <p>All Parquet writer cases:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">parquet</span><span class="o">-</span><span class="nb">format</span><span class="o">-</span><span class="n">benchmark</span> <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="n">BM_ParquetWrite</span> |
| </pre></div> |
| </div> |
| <p>Page-index filtering at 1% selectivity, on and off - compare <code class="docutils literal notranslate"><span class="pre">rows_read</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">read_bytes</span></code> and <code class="docutils literal notranslate"><span class="pre">ns_per_input_row</span></code> between the two:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">parquet</span><span class="o">-</span><span class="nb">format</span><span class="o">-</span><span class="n">benchmark</span> \ |
| <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="s1">'BM_ParquetRead_Filtered/keep_pct:1/'</span> |
| </pre></div> |
| </div> |
| <p>Read batch size sweep, repeated for a stable comparison:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">parquet</span><span class="o">-</span><span class="nb">format</span><span class="o">-</span><span class="n">benchmark</span> \ |
| <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="n">BM_ParquetRead_BatchSize</span> \ |
| <span class="o">--</span><span class="n">benchmark_repetitions</span><span class="o">=</span><span class="mi">5</span> \ |
| <span class="o">--</span><span class="n">benchmark_report_aggregates_only</span><span class="o">=</span><span class="n">true</span> |
| </pre></div> |
| </div> |
| <p>Null density on both sides, to see what definition levels cost:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">paimon</span><span class="o">-</span><span class="n">parquet</span><span class="o">-</span><span class="nb">format</span><span class="o">-</span><span class="n">benchmark</span> <span class="o">--</span><span class="n">benchmark_filter</span><span class="o">=</span><span class="s1">'Parquet(Write|Read)_Nulls'</span> |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer d-print-none"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="clean.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Clean Example</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="../api.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">API Reference</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <dialog id="pst-secondary-sidebar-modal"></dialog> |
| <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav id="pst-page-toc-nav" class="page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="pst-show_toc_level nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#build">Build</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#table-level-custom-options">Table-level Custom Options</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#examples">Examples</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#parquet-format-benchmark">Parquet Format Benchmark</a></li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/paimon-cpp/edit/main/docs/source/examples/benchmark.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script defer src="../_static/scripts/bootstrap.js?digest=cb8930eaaf36d9849b67"></script> |
| <script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=cb8930eaaf36d9849b67"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2026-2026 Apache Software Foundation. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 9.1.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| <!-- # L10n: Setting the PST URL as an argument as this does not need to be localized --> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.21.0. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
