| |
| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" /> |
| |
| <title>Arrow Columnar Format — Apache Arrow v17.0.0.dev52</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; |
| </script> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| |
| |
| <link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> |
| <script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script> |
| <script src="../_static/doctools.js"></script> |
| <script src="../_static/sphinx_highlight.js"></script> |
| <script src="../_static/clipboard.min.js"></script> |
| <script src="../_static/copybutton.js"></script> |
| <script src="../_static/design-tabs.js"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'format/Columnar';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.15.2'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/format/Columnar.html" /> |
| <link rel="icon" href="../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Canonical Extension Types" href="CanonicalExtensions.html" /> |
| <link rel="prev" title="Format Versioning and Stability" href="Versioning.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i> |
| Back to top |
| </button> |
| |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__primary" |
| id="__primary"/> |
| <label class="overlay overlay-primary" for="__primary"></label> |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__secondary" |
| id="__secondary"/> |
| <label class="overlay overlay-secondary" for="__secondary"></label> |
| |
| <div class="search-button__wrapper"> |
| <div class="search-button__overlay"></div> |
| <div class="search-button__search-container"> |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| id="search-input" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form></div> |
| </div> |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar"> |
| <div class="bd-header__inner bd-page-width"> |
| <label class="sidebar-toggle primary-toggle" for="__primary"> |
| <span class="fa-solid fa-bars"></span> |
| </label> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev52 - Home"/> |
| <script>document.write(`<img src="../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev52 - Home"/>`);</script> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| |
| <label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0"> |
| <span class="fa-solid fa-outdent"></span> |
| </label> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <div class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links-2" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="Versioning.html">Format Versioning and Stability</a></li> |
| <li class="toctree-l1 current active"><a class="current reference internal" href="#">Arrow Columnar Format</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="CanonicalExtensions.html">Canonical Extension Types</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Other.html">Other Data Structures</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="CDataInterface.html">The Arrow C data interface</a><input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="CDataInterface/PyCapsuleInterface.html">The Arrow PyCapsule Interface</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="CStreamInterface.html">The Arrow C stream interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="CDeviceDataInterface.html">The Arrow C Device data interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="FlightSql.html">Arrow Flight SQL</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="ADBC.html">ADBC: Arrow Database Connectivity</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/C.html">ADBC C API Specification</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/Go.html">ADBC Go API Specification</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/Java.html">ADBC Java API Specification</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="Changing.html">Changing the Apache Arrow Format Specification</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Integration.html">Integration Testing</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Glossary.html">Glossary</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| </div> |
| |
| <div id="rtd-footer-container"></div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| |
| |
| <nav aria-label="Breadcrumb"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Specifications</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page">Arrow...</li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="arrow-columnar-format"> |
| <span id="format-columnar"></span><h1>Arrow Columnar Format<a class="headerlink" href="#arrow-columnar-format" title="Permalink to this heading">#</a></h1> |
| <p><em>Version: 1.4</em></p> |
| <div class="admonition seealso"> |
| <p class="admonition-title">See also</p> |
| <p><a class="reference internal" href="Versioning.html#post-1-0-0-format-versions"><span class="std std-ref">Additions to the Arrow columnar format since version 1.0.0</span></a></p> |
| </div> |
| <p>The <strong>Arrow columnar format</strong> includes a language-agnostic in-memory |
| data structure specification, metadata serialization, and a protocol |
| for serialization and generic data transport.</p> |
| <p>This document is intended to provide adequate detail to create a new |
| implementation of the columnar format without the aid of an existing |
| implementation. We utilize Google’s <a class="reference external" href="http://github.com/google/flatbuffers">Flatbuffers</a> project for |
| metadata serialization, so it will be necessary to refer to the |
| project’s <a class="reference external" href="https://github.com/apache/arrow/tree/main/format">Flatbuffers protocol definition files</a> |
| while reading this document.</p> |
| <p>The columnar format has some key features:</p> |
| <ul class="simple"> |
| <li><p>Data adjacency for sequential access (scans)</p></li> |
| <li><p>O(1) (constant-time) random access</p></li> |
| <li><p>SIMD and vectorization-friendly</p></li> |
| <li><p>Relocatable without “pointer swizzling”, allowing for true zero-copy |
| access in shared memory</p></li> |
| </ul> |
| <p>The Arrow columnar format provides analytical performance and data |
| locality guarantees in exchange for comparatively more expensive |
| mutation operations. This document is concerned only with in-memory |
| data representation and serialization details; issues such as |
| coordinating mutation of data structures are left to be handled by |
| implementations.</p> |
| <section id="terminology"> |
| <h2>Terminology<a class="headerlink" href="#terminology" title="Permalink to this heading">#</a></h2> |
| <p>Since different projects have used different words to describe various |
| concepts, here is a small glossary to help disambiguate.</p> |
| <ul class="simple"> |
| <li><p><strong>Array</strong> or <strong>Vector</strong>: a sequence of values with known length all |
| having the same type. These terms are used interchangeably in |
| different Arrow implementations, but we use “array” in this |
| document.</p></li> |
| <li><p><strong>Slot</strong>: a single logical value in an array of some particular data type</p></li> |
| <li><p><strong>Buffer</strong> or <strong>Contiguous memory region</strong>: a sequential virtual |
| address space with a given length. Any byte can be reached via a |
| single pointer offset less than the region’s length.</p></li> |
| <li><p><strong>Physical Layout</strong>: The underlying memory layout for an array |
| without taking into account any value semantics. For example, a |
| 32-bit signed integer array and 32-bit floating point array have the |
| same layout.</p></li> |
| <li><p><strong>Parent</strong> and <strong>child arrays</strong>: names to express relationships |
| between physical value arrays in a nested type structure. For |
| example, a <code class="docutils literal notranslate"><span class="pre">List<T></span></code>-type parent array has a T-type array as its |
| child (see more on lists below).</p></li> |
| <li><p><strong>Primitive type</strong>: a data type having no child types. This includes |
| such types as fixed bit-width, variable-size binary, and null types.</p></li> |
| <li><p><strong>Nested type</strong>: a data type whose full structure depends on one or |
| more other child types. Two fully-specified nested types are equal |
| if and only if their child types are equal. For example, <code class="docutils literal notranslate"><span class="pre">List<U></span></code> |
| is distinct from <code class="docutils literal notranslate"><span class="pre">List<V></span></code> iff U and V are different types.</p></li> |
| <li><p><strong>Logical type</strong>: An application-facing semantic value type that is |
| implemented using some physical layout. For example, Decimal |
| values are stored as 16 bytes in a fixed-size binary |
| layout. Similarly, strings can be stored as <code class="docutils literal notranslate"><span class="pre">List<1-byte></span></code>. A |
| timestamp may be stored as 64-bit fixed-size layout.</p></li> |
| </ul> |
| </section> |
| <section id="physical-memory-layout"> |
| <span id="format-layout"></span><h2>Physical Memory Layout<a class="headerlink" href="#physical-memory-layout" title="Permalink to this heading">#</a></h2> |
| <p>Arrays are defined by a few pieces of metadata and data:</p> |
| <ul class="simple"> |
| <li><p>A logical data type.</p></li> |
| <li><p>A sequence of buffers.</p></li> |
| <li><p>A length as a 64-bit signed integer. Implementations are permitted |
| to be limited to 32-bit lengths, see more on this below.</p></li> |
| <li><p>A null count as a 64-bit signed integer.</p></li> |
| <li><p>An optional <strong>dictionary</strong>, for dictionary-encoded arrays.</p></li> |
| </ul> |
| <p>Nested arrays additionally have a sequence of one or more sets of |
| these items, called the <strong>child arrays</strong>.</p> |
| <p>Each logical data type has a well-defined physical layout. Here are |
| the different physical layouts defined by Arrow:</p> |
| <ul class="simple"> |
| <li><p><strong>Primitive (fixed-size)</strong>: a sequence of values each having the |
| same byte or bit width</p></li> |
| <li><p><strong>Variable-size Binary</strong>: a sequence of values each having a variable |
| byte length. Two variants of this layout are supported using 32-bit |
| and 64-bit length encoding.</p></li> |
| <li><p><strong>View of Variable-size Binary</strong>: a sequence of values each having a |
| variable byte length. In contrast to Variable-size Binary, the values |
| of this layout are distributed across potentially multiple buffers |
| instead of densely and sequentially packed in a single buffer.</p></li> |
| <li><p><strong>Fixed-size List</strong>: a nested layout where each value has the same |
| number of elements taken from a child data type.</p></li> |
| <li><p><strong>Variable-size List</strong>: a nested layout where each value is a |
| variable-length sequence of values taken from a child data type. Two |
| variants of this layout are supported using 32-bit and 64-bit length |
| encoding.</p></li> |
| <li><p><strong>View of Variable-size List</strong>: a nested layout where each value is a |
| variable-length sequence of values taken from a child data type. This |
| layout differs from <strong>Variable-size List</strong> by having an additional |
| buffer containing the sizes of each list value. This removes a constraint |
| on the offsets buffer — it does not need to be in order.</p></li> |
| <li><p><strong>Struct</strong>: a nested layout consisting of a collection of named |
| child <strong>fields</strong> each having the same length but possibly different |
| types.</p></li> |
| <li><p><strong>Sparse</strong> and <strong>Dense Union</strong>: a nested layout representing a |
| sequence of values, each of which can have type chosen from a |
| collection of child array types.</p></li> |
| <li><p><strong>Dictionary-Encoded</strong>: a layout consisting of a sequence of |
| integers (any bit-width) which represent indexes into a dictionary |
| which could be of any type.</p></li> |
| <li><p><strong>Run-End Encoded (REE)</strong>: a nested layout consisting of two child arrays, |
| one representing values, and one representing the logical index where |
| the run of a corresponding value ends.</p></li> |
| <li><p><strong>Null</strong>: a sequence of all null values, having null logical type</p></li> |
| </ul> |
| <p>The Arrow columnar memory layout only applies to <em>data</em> and not |
| <em>metadata</em>. Implementations are free to represent metadata in-memory |
| in whichever form is convenient for them. We handle metadata |
| <strong>serialization</strong> in an implementation-independent way using |
| <a class="reference external" href="http://github.com/google/flatbuffers">Flatbuffers</a>, detailed below.</p> |
| <section id="buffer-alignment-and-padding"> |
| <h3>Buffer Alignment and Padding<a class="headerlink" href="#buffer-alignment-and-padding" title="Permalink to this heading">#</a></h3> |
| <p>Implementations are recommended to allocate memory on aligned |
| addresses (multiple of 8- or 64-bytes) and pad (overallocate) to a |
| length that is a multiple of 8 or 64 bytes. When serializing Arrow |
| data for interprocess communication, these alignment and padding |
| requirements are enforced. If possible, we suggest that you prefer |
| using 64-byte alignment and padding. Unless otherwise noted, padded |
| bytes do not need to have a specific value.</p> |
| <p>The alignment requirement follows best practices for optimized memory |
| access:</p> |
| <ul class="simple"> |
| <li><p>Elements in numeric arrays will be guaranteed to be retrieved via aligned access.</p></li> |
| <li><p>On some architectures alignment can help limit partially used cache lines.</p></li> |
| </ul> |
| <p>The recommendation for 64 byte alignment comes from the <a class="reference external" href="https://software.intel.com/en-us/articles/practical-intel-avx-optimization-on-2nd-generation-intel-core-processors">Intel |
| performance guide</a> that recommends alignment of memory to match SIMD |
| register width. The specific padding length was chosen because it |
| matches the largest SIMD instruction registers available on widely |
| deployed x86 architecture (Intel AVX-512).</p> |
| <p>The recommended padding of 64 bytes allows for using <a class="reference external" href="https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-introduction-to-the-simd-data-layout-templates">SIMD</a> |
| instructions consistently in loops without additional conditional |
| checks. This should allow for simpler, efficient and CPU |
| cache-friendly code. In other words, we can load the entire 64-byte |
| buffer into a 512-bit wide SIMD register and get data-level |
| parallelism on all the columnar values packed into the 64-byte |
| buffer. Guaranteed padding can also allow certain compilers to |
| generate more optimized code directly (e.g. One can safely use Intel’s |
| <code class="docutils literal notranslate"><span class="pre">-qopt-assume-safe-padding</span></code>).</p> |
| </section> |
| <section id="array-lengths"> |
| <h3>Array lengths<a class="headerlink" href="#array-lengths" title="Permalink to this heading">#</a></h3> |
| <p>Array lengths are represented in the Arrow metadata as a 64-bit signed |
| integer. An implementation of Arrow is considered valid even if it only |
| supports lengths up to the maximum 32-bit signed integer, though. If using |
| Arrow in a multi-language environment, we recommend limiting lengths to |
| 2 <sup>31</sup> - 1 elements or less. Larger data sets can be represented using |
| multiple array chunks.</p> |
| </section> |
| <section id="null-count"> |
| <h3>Null count<a class="headerlink" href="#null-count" title="Permalink to this heading">#</a></h3> |
| <p>The number of null value slots is a property of the physical array and |
| considered part of the data structure. The null count is represented |
| in the Arrow metadata as a 64-bit signed integer, as it may be as |
| large as the array length.</p> |
| </section> |
| <section id="validity-bitmaps"> |
| <h3>Validity bitmaps<a class="headerlink" href="#validity-bitmaps" title="Permalink to this heading">#</a></h3> |
| <p>Any value in an array may be semantically null, whether primitive or nested |
| type.</p> |
| <p>All array types, with the exception of union types (more on these later), |
| utilize a dedicated memory buffer, known as the validity (or “null”) bitmap, to |
| encode the nullness or non-nullness of each value slot. The validity bitmap |
| must be large enough to have at least 1 bit for each array slot.</p> |
| <p>Whether any array slot is valid (non-null) is encoded in the respective bits of |
| this bitmap. A 1 (set bit) for index <code class="docutils literal notranslate"><span class="pre">j</span></code> indicates that the value is not null, |
| while a 0 (bit not set) indicates that it is null. Bitmaps are to be |
| initialized to be all unset at allocation time (this includes padding):</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">is_valid</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> <span class="o">-></span> <span class="n">bitmap</span><span class="p">[</span><span class="n">j</span> <span class="o">/</span> <span class="mi">8</span><span class="p">]</span> <span class="o">&</span> <span class="p">(</span><span class="mi">1</span> <span class="o"><<</span> <span class="p">(</span><span class="n">j</span> <span class="o">%</span> <span class="mi">8</span><span class="p">))</span> |
| </pre></div> |
| </div> |
| <p>We use <a class="reference external" href="https://en.wikipedia.org/wiki/Bit_numbering">least-significant bit (LSB) numbering</a> (also known as |
| bit-endianness). This means that within a group of 8 bits, we read |
| right-to-left:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> |
| |
| <span class="n">bitmap</span> |
| <span class="n">j</span> <span class="n">mod</span> <span class="mi">8</span> <span class="mi">7</span> <span class="mi">6</span> <span class="mi">5</span> <span class="mi">4</span> <span class="mi">3</span> <span class="mi">2</span> <span class="mi">1</span> <span class="mi">0</span> |
| <span class="mi">0</span> <span class="mi">0</span> <span class="mi">1</span> <span class="mi">0</span> <span class="mi">1</span> <span class="mi">0</span> <span class="mi">1</span> <span class="mi">1</span> |
| </pre></div> |
| </div> |
| <p>Arrays having a 0 null count may choose to not allocate the validity |
| bitmap; how this is represented depends on the implementation (for |
| example, a C++ implementation may represent such an “absent” validity |
| bitmap using a NULL pointer). Implementations may choose to always allocate |
| a validity bitmap anyway as a matter of convenience. Consumers of Arrow |
| arrays should be ready to handle those two possibilities.</p> |
| <p>Nested type arrays (except for union types as noted above) have their own |
| top-level validity bitmap and null count, regardless of the null count and |
| valid bits of their child arrays.</p> |
| <p>Array slots which are null are not required to have a particular value; |
| any “masked” memory can have any value and need not be zeroed, though |
| implementations frequently choose to zero memory for null values.</p> |
| </section> |
| <section id="fixed-size-primitive-layout"> |
| <h3>Fixed-size Primitive Layout<a class="headerlink" href="#fixed-size-primitive-layout" title="Permalink to this heading">#</a></h3> |
| <p>A primitive value array represents an array of values each having the |
| same physical slot width typically measured in bytes, though the spec |
| also provides for bit-packed types (e.g. boolean values encoded in |
| bits).</p> |
| <p>Internally, the array contains a contiguous memory buffer whose total |
| size is at least as large as the slot width multiplied by the array |
| length. For bit-packed types, the size is rounded up to the nearest |
| byte.</p> |
| <p>The associated validity bitmap is contiguously allocated (as described |
| above) but does not need to be adjacent in memory to the values |
| buffer.</p> |
| <p><strong>Example Layout: Int32 Array</strong></p> |
| <p>For example a primitive array of int32s:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>Would look like:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00011101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-------------|-------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p><strong>Example Layout: Non-null int32 Array</strong></p> |
| <p><code class="docutils literal notranslate"><span class="pre">[1,</span> <span class="pre">2,</span> <span class="pre">3,</span> <span class="pre">4,</span> <span class="pre">8]</span></code> has two possible layouts:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00011111</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-------------|-------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p>or with the bitmap elided:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span> <span class="mi">5</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="nb">bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-------------|-------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">8</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="variable-size-binary-layout"> |
| <h3>Variable-size Binary Layout<a class="headerlink" href="#variable-size-binary-layout" title="Permalink to this heading">#</a></h3> |
| <p>Each value in this layout consists of 0 or more bytes. While primitive |
| arrays have a single values buffer, variable-size binary have an |
| <strong>offsets</strong> buffer and <strong>data</strong> buffer.</p> |
| <p>The offsets buffer contains <cite>length + 1</cite> signed integers (either |
| 32-bit or 64-bit, depending on the logical type), which encode the |
| start position of each slot in the data buffer. The length of the |
| value in each slot is computed using the difference between the offset |
| at that slot’s index and the subsequent offset. For example, the |
| position and length of slot j is computed as:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">slot_position</span> <span class="o">=</span> <span class="n">offsets</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> |
| <span class="n">slot_length</span> <span class="o">=</span> <span class="n">offsets</span><span class="p">[</span><span class="n">j</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="n">offsets</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> <span class="o">//</span> <span class="p">(</span><span class="k">for</span> <span class="mi">0</span> <span class="o"><=</span> <span class="n">j</span> <span class="o"><</span> <span class="n">length</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>It should be noted that a null value may have a positive slot length. |
| That is, a null value may occupy a <strong>non-empty</strong> memory space in the data |
| buffer. When this is true, the content of the corresponding memory space |
| is undefined.</p> |
| <p>Offsets must be monotonically increasing, that is <code class="docutils literal notranslate"><span class="pre">offsets[j+1]</span> <span class="pre">>=</span> <span class="pre">offsets[j]</span></code> |
| for <code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre"><=</span> <span class="pre">j</span> <span class="pre"><</span> <span class="pre">length</span></code>, even for null slots. This property ensures the |
| location for all values is valid and well defined.</p> |
| <p>Generally the first slot in the offsets array is 0, and the last slot |
| is the length of the values array. When serializing this layout, we |
| recommend normalizing the offsets to start at 0.</p> |
| <p><strong>Example Layout: ``VarBinary``</strong></p> |
| <p><code class="docutils literal notranslate"><span class="pre">['joe',</span> <span class="pre">null,</span> <span class="pre">null,</span> <span class="pre">'mark']</span></code></p> |
| <p>will be represented as follows:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">2</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00001001</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Offsets</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|----------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">7</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">6</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">7</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|----------------|-----------------------|</span> |
| <span class="o">|</span> <span class="n">joemark</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="variable-size-binary-view-layout"> |
| <span id="id1"></span><h3>Variable-size Binary View Layout<a class="headerlink" href="#variable-size-binary-view-layout" title="Permalink to this heading">#</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version Arrow: </span>Columnar Format 1.4</p> |
| </div> |
| <p>Each value in this layout consists of 0 or more bytes. These bytes’ |
| locations are indicated using a <strong>views</strong> buffer, which may point to one |
| of potentially several <strong>data</strong> buffers or may contain the characters |
| inline.</p> |
| <p>The views buffer contains <cite>length</cite> view structures with the following layout:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Short</span> <span class="n">strings</span><span class="p">,</span> <span class="n">length</span> <span class="o"><=</span> <span class="mi">12</span> |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> |
| <span class="o">|------------|---------------------------------------|</span> |
| <span class="o">|</span> <span class="n">length</span> <span class="o">|</span> <span class="n">data</span> <span class="p">(</span><span class="n">padded</span> <span class="k">with</span> <span class="mi">0</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Long</span> <span class="n">strings</span><span class="p">,</span> <span class="n">length</span> <span class="o">></span> <span class="mi">12</span> |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> |
| <span class="o">|------------|------------|------------|-------------|</span> |
| <span class="o">|</span> <span class="n">length</span> <span class="o">|</span> <span class="n">prefix</span> <span class="o">|</span> <span class="n">buf</span><span class="o">.</span> <span class="n">index</span> <span class="o">|</span> <span class="n">offset</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p>In both the long and short string cases, the first four bytes encode the |
| length of the string and can be used to determine how the rest of the view |
| should be interpreted.</p> |
| <p>In the short string case the string’s bytes are inlined — stored inside the |
| view itself, in the twelve bytes which follow the length. Any remaining bytes |
| after the string itself are padded with <cite>0</cite>.</p> |
| <p>In the long string case, a buffer index indicates which data buffer |
| stores the data bytes and an offset indicates where in that buffer the |
| data bytes begin. Buffer index 0 refers to the first data buffer, IE |
| the first buffer <strong>after</strong> the validity buffer and the views buffer. |
| The half-open range <code class="docutils literal notranslate"><span class="pre">[offset,</span> <span class="pre">offset</span> <span class="pre">+</span> <span class="pre">length)</span></code> must be entirely contained |
| within the indicated buffer. A copy of the first four bytes of the string is |
| stored inline in the prefix, after the length. This prefix enables a |
| profitable fast path for string comparisons, which are frequently determined |
| within the first four bytes.</p> |
| <p>All integers (length, buffer index, and offset) are signed.</p> |
| <p>This layout is adapted from TU Munich’s <a class="reference external" href="https://db.in.tum.de/~freitag/papers/p29-neumann-cidr20.pdf">UmbraDB</a>.</p> |
| <p>Note that this layout uses one additional buffer to store the variadic buffer |
| lengths in the <a class="reference internal" href="CDataInterface.html#c-data-interface-binary-view-arrays"><span class="std std-ref">Arrow C data interface</span></a>.</p> |
| </section> |
| <section id="variable-size-list-layout"> |
| <span id="id2"></span><h3>Variable-size List Layout<a class="headerlink" href="#variable-size-list-layout" title="Permalink to this heading">#</a></h3> |
| <p>List is a nested type which is semantically similar to variable-size |
| binary. There are two list layout variations — “list” and “list-view” — |
| and each variation can be delimited by either 32-bit or 64-bit offsets |
| integers.</p> |
| <section id="list-layout"> |
| <h4>List Layout<a class="headerlink" href="#list-layout" title="Permalink to this heading">#</a></h4> |
| <p>The List layout is defined by two buffers, a validity bitmap and an offsets |
| buffer, and a child array. The offsets are the same as in the |
| variable-size binary case, and both 32-bit and 64-bit signed integer |
| offsets are supported options for the offsets. Rather than referencing |
| an additional data buffer, instead these offsets reference the child |
| array.</p> |
| <p>Similar to the layout of variable-size binary, a null value may |
| correspond to a <strong>non-empty</strong> segment in the child array. When this is |
| true, the content of the corresponding segment can be arbitrary.</p> |
| <p>A list type is specified like <code class="docutils literal notranslate"><span class="pre">List<T></span></code>, where <code class="docutils literal notranslate"><span class="pre">T</span></code> is any type |
| (primitive or nested). In these examples we use 32-bit offsets where |
| the 64-bit offset version would be denoted by <code class="docutils literal notranslate"><span class="pre">LargeList<T></span></code>.</p> |
| <p><strong>Example Layout: ``List<Int8>`` Array</strong></p> |
| <p>We illustrate an example of <code class="docutils literal notranslate"><span class="pre">List<Int8></span></code> with length 4 having values:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[[</span><span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span><span class="p">],</span> <span class="n">null</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span><span class="p">],</span> <span class="p">[]]</span> |
| </pre></div> |
| </div> |
| <p>will have the following representation:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00001101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Offsets</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int32</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">array</span> <span class="p">(</span><span class="n">Int8Array</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int8</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">6</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">7</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p><strong>Example Layout: ``List<List<Int8>>``</strong></p> |
| <p><code class="docutils literal notranslate"><span class="pre">[[[1,</span> <span class="pre">2],</span> <span class="pre">[3,</span> <span class="pre">4]],</span> <span class="pre">[[5,</span> <span class="pre">6,</span> <span class="pre">7],</span> <span class="pre">null,</span> <span class="pre">[8]],</span> <span class="pre">[[9,</span> <span class="pre">10]]]</span></code></p> |
| <p>will be represented as follows:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>* Length 3 |
| * Nulls count: 0 |
| * Validity bitmap buffer: Not required |
| * Offsets buffer (int32) |
| |
| | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-63 | |
| |------------|------------|------------|-------------|-----------------------| |
| | 0 | 2 | 5 | 6 | unspecified (padding) | |
| |
| * Values array (`List<Int8>`) |
| * Length: 6, Null count: 1 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-------------| |
| | 00110111 | 0 (padding) | |
| |
| * Offsets buffer (int32) |
| |
| | Bytes 0-27 | Bytes 28-63 | |
| |----------------------|-----------------------| |
| | 0, 2, 4, 7, 7, 8, 10 | unspecified (padding) | |
| |
| * Values array (Int8): |
| * Length: 10, Null count: 0 |
| * Validity bitmap buffer: Not required |
| |
| | Bytes 0-9 | Bytes 10-63 | |
| |-------------------------------|-----------------------| |
| | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 | unspecified (padding) | |
| </pre></div> |
| </div> |
| </section> |
| <section id="listview-layout"> |
| <span id="id3"></span><h4>ListView Layout<a class="headerlink" href="#listview-layout" title="Permalink to this heading">#</a></h4> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version Arrow: </span>Columnar Format 1.4</p> |
| </div> |
| <p>The ListView layout is defined by three buffers: a validity bitmap, an offsets |
| buffer, and an additional sizes buffer. Sizes and offsets have the identical bit |
| width and both 32-bit and 64-bit signed integer options are supported.</p> |
| <p>As in the List layout, the offsets encode the start position of each slot in the |
| child array. In contrast to the List layout, list lengths are stored explicitly |
| in the sizes buffer instead of inferred. This allows offsets to be out of order. |
| Elements of the child array do not have to be stored in the same order they |
| logically appear in the list elements of the parent array.</p> |
| <p>Every list-view value, including null values, has to guarantee the following |
| invariants:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="mi">0</span> <span class="o"><=</span> <span class="n">offsets</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o"><=</span> <span class="n">length</span> <span class="n">of</span> <span class="n">the</span> <span class="n">child</span> <span class="n">array</span> |
| <span class="mi">0</span> <span class="o"><=</span> <span class="n">offsets</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">+</span> <span class="n">size</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o"><=</span> <span class="n">length</span> <span class="n">of</span> <span class="n">the</span> <span class="n">child</span> <span class="n">array</span> |
| </pre></div> |
| </div> |
| <p>A list-view type is specified like <code class="docutils literal notranslate"><span class="pre">ListView<T></span></code>, where <code class="docutils literal notranslate"><span class="pre">T</span></code> is any type |
| (primitive or nested). In these examples we use 32-bit offsets and sizes where |
| the 64-bit version would be denoted by <code class="docutils literal notranslate"><span class="pre">LargeListView<T></span></code>.</p> |
| <p><strong>Example Layout: ``ListView<Int8>`` Array</strong></p> |
| <p>We illustrate an example of <code class="docutils literal notranslate"><span class="pre">ListView<Int8></span></code> with length 4 having values:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[[</span><span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span><span class="p">],</span> <span class="n">null</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span><span class="p">],</span> <span class="p">[]]</span> |
| </pre></div> |
| </div> |
| <p>It may have the following representation:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00001101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Offsets</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int32</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Sizes</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int32</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">array</span> <span class="p">(</span><span class="n">Int8Array</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int8</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">6</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">7</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| <p><strong>Example Layout: ``ListView<Int8>`` Array</strong></p> |
| <p>We continue with the <code class="docutils literal notranslate"><span class="pre">ListView<Int8></span></code> type, but this instance illustrates out |
| of order offsets and sharing of child array values. It is an array with length 5 |
| having logical values:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[[</span><span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span><span class="p">],</span> <span class="n">null</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span><span class="p">],</span> <span class="p">[],</span> <span class="p">[</span><span class="mi">50</span><span class="p">,</span> <span class="mi">12</span><span class="p">]]</span> |
| </pre></div> |
| </div> |
| <p>It may have the following representation:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00011101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Offsets</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int32</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Sizes</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int32</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">19</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">20</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------|-------------|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">3</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">array</span> <span class="p">(</span><span class="n">Int8Array</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">buffer</span> <span class="p">(</span><span class="n">int8</span><span class="p">)</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">6</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">7</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|------------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">127</span><span class="p">,</span> <span class="mi">127</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">12</span><span class="p">,</span> <span class="o">-</span><span class="mi">7</span><span class="p">,</span> <span class="mi">25</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| <section id="fixed-size-list-layout"> |
| <h3>Fixed-Size List Layout<a class="headerlink" href="#fixed-size-list-layout" title="Permalink to this heading">#</a></h3> |
| <p>Fixed-Size List is a nested type in which each array slot contains a |
| fixed-size sequence of values all having the same type.</p> |
| <p>A fixed size list type is specified like <code class="docutils literal notranslate"><span class="pre">FixedSizeList<T>[N]</span></code>, |
| where <code class="docutils literal notranslate"><span class="pre">T</span></code> is any type (primitive or nested) and <code class="docutils literal notranslate"><span class="pre">N</span></code> is a 32-bit |
| signed integer representing the length of the lists.</p> |
| <p>A fixed size list array is represented by a values array, which is a |
| child array of type T. T may also be a nested type. The value in slot |
| <code class="docutils literal notranslate"><span class="pre">j</span></code> of a fixed size list array is stored in an <code class="docutils literal notranslate"><span class="pre">N</span></code>-long slice of |
| the values array, starting at an offset of <code class="docutils literal notranslate"><span class="pre">j</span> <span class="pre">*</span> <span class="pre">N</span></code>.</p> |
| <p><strong>Example Layout: ``FixedSizeList<byte>[4]`` Array</strong></p> |
| <p>Here we illustrate <code class="docutils literal notranslate"><span class="pre">FixedSizeList<byte>[4]</span></code>.</p> |
| <p>For an array of length 4 with respective values:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[[</span><span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">12</span><span class="p">],</span> <span class="n">null</span><span class="p">,</span> <span class="p">[</span><span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">25</span><span class="p">],</span> <span class="p">[</span><span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]]</span> |
| </pre></div> |
| </div> |
| <p>will have the following representation:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00001101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">array</span> <span class="p">(</span><span class="n">byte</span> <span class="n">array</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">16</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> |
| <span class="o">|-----------------|-------------|---------------------------------|</span> |
| <span class="o">|</span> <span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">12</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> <span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">25</span><span class="p">,</span> <span class="mi">192</span><span class="p">,</span> <span class="mi">168</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="struct-layout"> |
| <h3>Struct Layout<a class="headerlink" href="#struct-layout" title="Permalink to this heading">#</a></h3> |
| <p>A struct is a nested type parameterized by an ordered sequence of |
| types (which can all be distinct), called its fields. Each field must |
| have a UTF8-encoded name, and these field names are part of the type |
| metadata.</p> |
| <p>Physically, a struct array has one child array for each field. The |
| child arrays are independent and need not be adjacent to each other in |
| memory. A struct array also has a validity bitmap to encode top-level |
| validity information.</p> |
| <p>For example, the struct (field names shown here as strings for illustration |
| purposes):</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Struct</span> <span class="o"><</span> |
| <span class="n">name</span><span class="p">:</span> <span class="n">VarBinary</span> |
| <span class="n">age</span><span class="p">:</span> <span class="n">Int32</span> |
| <span class="o">></span> |
| </pre></div> |
| </div> |
| <p>has two child arrays, one <code class="docutils literal notranslate"><span class="pre">VarBinary</span></code> array (using variable-size binary |
| layout) and one 4-byte primitive value array having <code class="docutils literal notranslate"><span class="pre">Int32</span></code> logical |
| type.</p> |
| <p><strong>Example Layout: ``Struct<VarBinary, Int32>``</strong></p> |
| <p>The layout for <code class="docutils literal notranslate"><span class="pre">[{'joe',</span> <span class="pre">1},</span> <span class="pre">{null,</span> <span class="pre">2},</span> <span class="pre">null,</span> <span class="pre">{'mark',</span> <span class="pre">4}]</span></code>, having |
| child arrays <code class="docutils literal notranslate"><span class="pre">['joe',</span> <span class="pre">null,</span> <span class="pre">'alice',</span> <span class="pre">'mark']</span></code> and <code class="docutils literal notranslate"><span class="pre">[1,</span> <span class="pre">2,</span> <span class="pre">null,</span> <span class="pre">4]</span></code> |
| would be:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>* Length: 4, Null count: 1 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00001011 | 0 (padding) | |
| |
| * Children arrays: |
| * field-0 array (`VarBinary`): |
| * Length: 4, Null count: 1 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00001101 | 0 (padding) | |
| |
| * Offsets buffer: |
| |
| | Bytes 0-19 | Bytes 20-63 | |
| |----------------|-----------------------| |
| | 0, 3, 3, 8, 12 | unspecified (padding) | |
| |
| * Value buffer: |
| |
| | Bytes 0-11 | Bytes 12-63 | |
| |----------------|-----------------------| |
| | joealicemark | unspecified (padding) | |
| |
| * field-1 array (int32 array): |
| * Length: 4, Null count: 1 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00001011 | 0 (padding) | |
| |
| * Value Buffer: |
| |
| | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-63 | |
| |-------------|-------------|-------------|-------------|-----------------------| |
| | 1 | 2 | unspecified | 4 | unspecified (padding) | |
| </pre></div> |
| </div> |
| <section id="struct-validity"> |
| <h4>Struct Validity<a class="headerlink" href="#struct-validity" title="Permalink to this heading">#</a></h4> |
| <p>A struct array has its own validity bitmap that is independent of its |
| child arrays’ validity bitmaps. The validity bitmap for the struct |
| array might indicate a null when one or more of its child arrays has |
| a non-null value in its corresponding slot; or conversely, a child |
| array might indicate a null in its validity bitmap while the struct array’s |
| validity bitmap shows a non-null value.</p> |
| <p>Therefore, to know whether a particular child entry is valid, one must |
| take the logical AND of the corresponding bits in the two validity bitmaps |
| (the struct array’s and the child array’s).</p> |
| <p>This is illustrated in the example above, one of the child arrays has a |
| valid entry <code class="docutils literal notranslate"><span class="pre">'alice'</span></code> for the null struct but it is “hidden” by the |
| struct array’s validity bitmap. However, when treated independently, |
| corresponding entries of the children array will be non-null.</p> |
| </section> |
| </section> |
| <section id="union-layout"> |
| <h3>Union Layout<a class="headerlink" href="#union-layout" title="Permalink to this heading">#</a></h3> |
| <p>A union is defined by an ordered sequence of types; each slot in the |
| union can have a value chosen from these types. The types are named |
| like a struct’s fields, and the names are part of the type metadata.</p> |
| <p>Unlike other data types, unions do not have their own validity bitmap. Instead, |
| the nullness of each slot is determined exclusively by the child arrays which |
| are composed to create the union.</p> |
| <p>We define two distinct union types, “dense” and “sparse”, that are |
| optimized for different use cases.</p> |
| <section id="dense-union"> |
| <h4>Dense Union<a class="headerlink" href="#dense-union" title="Permalink to this heading">#</a></h4> |
| <p>Dense union represents a mixed-type array with 5 bytes of overhead for |
| each value. Its physical layout is as follows:</p> |
| <ul class="simple"> |
| <li><p>One child array for each type</p></li> |
| <li><p>Types buffer: A buffer of 8-bit signed integers. Each type in the |
| union has a corresponding type id whose values are found in this |
| buffer. A union with more than 127 possible types can be modeled as |
| a union of unions.</p></li> |
| <li><p>Offsets buffer: A buffer of signed Int32 values indicating the |
| relative offset into the respective child array for the type in a |
| given slot. The respective offsets for each child value array must |
| be in order / increasing.</p></li> |
| </ul> |
| <p><strong>Example Layout: ``DenseUnion<f: Float32, i: Int32>``</strong></p> |
| <p>For the union array:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="n">f</span><span class="o">=</span><span class="mf">1.2</span><span class="p">},</span> <span class="n">null</span><span class="p">,</span> <span class="p">{</span><span class="n">f</span><span class="o">=</span><span class="mf">3.4</span><span class="p">},</span> <span class="p">{</span><span class="n">i</span><span class="o">=</span><span class="mi">5</span><span class="p">}]</span> |
| </pre></div> |
| </div> |
| <p>will have the following layout:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Types</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="o">|</span> <span class="n">Byte</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">Byte</span> <span class="mi">2</span> <span class="o">|</span> <span class="n">Byte</span> <span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|----------|-------------|----------|----------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Offset</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">15</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">16</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-----------|-------------|------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="mi">1</span> <span class="o">|</span> <span class="mi">2</span> <span class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Children</span> <span class="n">arrays</span><span class="p">:</span> |
| <span class="o">*</span> <span class="n">Field</span><span class="o">-</span><span class="mi">0</span> <span class="n">array</span> <span class="p">(</span><span class="n">f</span><span class="p">:</span> <span class="n">Float32</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="mi">00000101</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|----------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mf">1.2</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mf">3.4</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| |
| <span class="o">*</span> <span class="n">Field</span><span class="o">-</span><span class="mi">1</span> <span class="n">array</span> <span class="p">(</span><span class="n">i</span><span class="p">:</span> <span class="n">Int32</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> |
| |
| <span class="o">*</span> <span class="n">Value</span> <span class="n">Buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-----------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="sparse-union"> |
| <h4>Sparse Union<a class="headerlink" href="#sparse-union" title="Permalink to this heading">#</a></h4> |
| <p>A sparse union has the same structure as a dense union, with the omission of |
| the offsets array. In this case, the child arrays are each equal in length to |
| the length of the union.</p> |
| <p>While a sparse union may use significantly more space compared with a |
| dense union, it has some advantages that may be desirable in certain |
| use cases:</p> |
| <ul class="simple"> |
| <li><p>A sparse union is more amenable to vectorized expression evaluation in some use cases.</p></li> |
| <li><p>Equal-length arrays can be interpreted as a union by only defining the types array.</p></li> |
| </ul> |
| <p><strong>Example layout: ``SparseUnion<i: Int32, f: Float32, s: VarBinary>``</strong></p> |
| <p>For the union array:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="n">i</span><span class="o">=</span><span class="mi">5</span><span class="p">},</span> <span class="p">{</span><span class="n">f</span><span class="o">=</span><span class="mf">1.2</span><span class="p">},</span> <span class="p">{</span><span class="n">s</span><span class="o">=</span><span class="s1">'joe'</span><span class="p">},</span> <span class="p">{</span><span class="n">f</span><span class="o">=</span><span class="mf">3.4</span><span class="p">},</span> <span class="p">{</span><span class="n">i</span><span class="o">=</span><span class="mi">4</span><span class="p">},</span> <span class="p">{</span><span class="n">s</span><span class="o">=</span><span class="s1">'mark'</span><span class="p">}]</span> |
| </pre></div> |
| </div> |
| <p>will have the following layout:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>* Length: 6, Null count: 0 |
| * Types buffer: |
| |
| | Byte 0 | Byte 1 | Byte 2 | Byte 3 | Byte 4 | Byte 5 | Bytes 6-63 | |
| |------------|-------------|-------------|-------------|-------------|--------------|-----------------------| |
| | 0 | 1 | 2 | 1 | 0 | 2 | unspecified (padding) | |
| |
| * Children arrays: |
| |
| * i (Int32): |
| * Length: 6, Null count: 4 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00010001 | 0 (padding) | |
| |
| * Value buffer: |
| |
| | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-63 | |
| |-------------|-------------|-------------|-------------|-------------|--------------|-----------------------| |
| | 5 | unspecified | unspecified | unspecified | 4 | unspecified | unspecified (padding) | |
| |
| * f (Float32): |
| * Length: 6, Null count: 4 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00001010 | 0 (padding) | |
| |
| * Value buffer: |
| |
| | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-63 | |
| |--------------|-------------|-------------|-------------|-------------|-------------|-----------------------| |
| | unspecified | 1.2 | unspecified | 3.4 | unspecified | unspecified | unspecified (padding) | |
| |
| * s (`VarBinary`) |
| * Length: 6, Null count: 4 |
| * Validity bitmap buffer: |
| |
| | Byte 0 (validity bitmap) | Bytes 1-63 | |
| |--------------------------|-----------------------| |
| | 00100100 | 0 (padding) | |
| |
| * Offsets buffer (Int32) |
| |
| | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-27 | Bytes 28-63 | |
| |------------|-------------|-------------|-------------|-------------|-------------|-------------|------------------------| |
| | 0 | 0 | 0 | 3 | 3 | 3 | 7 | unspecified (padding) | |
| |
| * Values buffer: |
| |
| | Bytes 0-6 | Bytes 7-63 | |
| |------------|-----------------------| |
| | joemark | unspecified (padding) | |
| </pre></div> |
| </div> |
| <p>Only the slot in the array corresponding to the type index is considered. All |
| “unselected” values are ignored and could be any semantically correct array |
| value.</p> |
| </section> |
| </section> |
| <section id="null-layout"> |
| <h3>Null Layout<a class="headerlink" href="#null-layout" title="Permalink to this heading">#</a></h3> |
| <p>We provide a simplified memory-efficient layout for the Null data type |
| where all values are null. In this case no memory buffers are |
| allocated.</p> |
| </section> |
| <section id="dictionary-encoded-layout"> |
| <span id="id4"></span><h3>Dictionary-encoded Layout<a class="headerlink" href="#dictionary-encoded-layout" title="Permalink to this heading">#</a></h3> |
| <p>Dictionary encoding is a data representation technique to represent |
| values by integers referencing a <strong>dictionary</strong> usually consisting of |
| unique values. It can be effective when you have data with many |
| repeated values.</p> |
| <p>Any array can be dictionary-encoded. The dictionary is stored as an optional |
| property of an array. When a field is dictionary encoded, the values are |
| represented by an array of non-negative integers representing the index of the |
| value in the dictionary. The memory layout for a dictionary-encoded array is |
| the same as that of a primitive integer layout. The dictionary is handled as a |
| separate columnar array with its own respective layout.</p> |
| <p>As an example, you could have the following data:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">type</span><span class="p">:</span> <span class="n">VarBinary</span> |
| |
| <span class="p">[</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>In dictionary-encoded form, this could appear as:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">data</span> <span class="n">VarBinary</span> <span class="p">(</span><span class="n">dictionary</span><span class="o">-</span><span class="n">encoded</span><span class="p">)</span> |
| <span class="n">index_type</span><span class="p">:</span> <span class="n">Int32</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span> |
| |
| <span class="n">dictionary</span> |
| <span class="nb">type</span><span class="p">:</span> <span class="n">VarBinary</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>Note that a dictionary is permitted to contain duplicate values or |
| nulls:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">data</span> <span class="n">VarBinary</span> <span class="p">(</span><span class="n">dictionary</span><span class="o">-</span><span class="n">encoded</span><span class="p">)</span> |
| <span class="n">index_type</span><span class="p">:</span> <span class="n">Int32</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span> |
| |
| <span class="n">dictionary</span> |
| <span class="nb">type</span><span class="p">:</span> <span class="n">VarBinary</span> |
| <span class="n">values</span><span class="p">:</span> <span class="p">[</span><span class="s1">'foo'</span><span class="p">,</span> <span class="s1">'bar'</span><span class="p">,</span> <span class="s1">'baz'</span><span class="p">,</span> <span class="s1">'foo'</span><span class="p">,</span> <span class="n">null</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>The null count of such arrays is dictated only by the validity bitmap |
| of its indices, irrespective of any null values in the dictionary.</p> |
| <p>Since unsigned integers can be more difficult to work with in some cases |
| (e.g. in the JVM), we recommend preferring signed integers over unsigned |
| integers for representing dictionary indices. Additionally, we recommend |
| avoiding using 64-bit unsigned integer indices unless they are required by an |
| application.</p> |
| <p>We discuss dictionary encoding as it relates to serialization further |
| below.</p> |
| </section> |
| <section id="run-end-encoded-layout"> |
| <span id="id5"></span><h3>Run-End Encoded Layout<a class="headerlink" href="#run-end-encoded-layout" title="Permalink to this heading">#</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version Arrow: </span>Columnar Format 1.3</p> |
| </div> |
| <p>Run-end encoding (REE) is a variation of run-length encoding (RLE). These |
| encodings are well-suited for representing data containing sequences of the |
| same value, called runs. In run-end encoding, each run is represented as a |
| value and an integer giving the index in the array where the run ends.</p> |
| <p>Any array can be run-end encoded. A run-end encoded array has no buffers |
| by itself, but has two child arrays. The first child array, called the run ends array, |
| holds either 16, 32, or 64-bit signed integers. The actual values of each run |
| are held in the second child array. |
| For the purposes of determining field names and schemas, these child arrays |
| are prescribed the standard names of <strong>run_ends</strong> and <strong>values</strong> respectively.</p> |
| <p>The values in the first child array represent the accumulated length of all runs |
| from the first to the current one, i.e. the logical index where the |
| current run ends. This allows relatively efficient random access from a logical |
| index using binary search. The length of an individual run can be determined by |
| subtracting two adjacent values. (Contrast this with run-length encoding, in |
| which the lengths of the runs are represented directly, and in which random |
| access is less efficient.)</p> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Because the <code class="docutils literal notranslate"><span class="pre">run_ends</span></code> child array cannot have nulls, it’s reasonable |
| to consider why the <code class="docutils literal notranslate"><span class="pre">run_ends</span></code> are a child array instead of just a |
| buffer, like the offsets for a <a class="reference internal" href="#variable-size-list-layout"><span class="std std-ref">Variable-size List Layout</span></a>. This |
| layout was considered, but it was decided to use the child arrays.</p> |
| <p>Child arrays allow us to keep the “logical length” (the decoded length) |
| associated with the parent array and the “physical length” (the number |
| of run ends) associated with the child arrays. If <code class="docutils literal notranslate"><span class="pre">run_ends</span></code> was a |
| buffer in the parent array then the size of the buffer would be unrelated |
| to the length of the array and this would be confusing.</p> |
| </div> |
| <p>A run must have a length of at least 1. This means the values in the |
| run ends array all are positive and in strictly ascending order. A run end cannot be |
| null.</p> |
| <p>The REE parent has no validity bitmap, and it’s null count field should always be 0. |
| Null values are encoded as runs with the value null.</p> |
| <p>As an example, you could have the following data:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">type</span><span class="p">:</span> <span class="n">Float32</span> |
| <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="n">null</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">]</span> |
| </pre></div> |
| </div> |
| <p>In Run-end-encoded form, this could appear as:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> |
| <span class="o">*</span> <span class="n">Child</span> <span class="n">Arrays</span><span class="p">:</span> |
| |
| <span class="o">*</span> <span class="n">run_ends</span> <span class="p">(</span><span class="n">Int32</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">0</span> <span class="p">(</span><span class="n">Run</span> <span class="n">Ends</span> <span class="n">cannot</span> <span class="n">be</span> <span class="n">null</span><span class="p">)</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> <span class="n">Not</span> <span class="n">required</span> <span class="p">(</span><span class="k">if</span> <span class="n">it</span> <span class="n">exists</span><span class="p">,</span> <span class="n">it</span> <span class="n">should</span> <span class="n">be</span> <span class="nb">all</span> <span class="mi">1</span><span class="n">s</span><span class="p">)</span> |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">buffer</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">4</span> <span class="o">|</span> <span class="mi">6</span> <span class="o">|</span> <span class="mi">7</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">values</span> <span class="p">(</span><span class="n">Float32</span><span class="p">):</span> |
| <span class="o">*</span> <span class="n">Length</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="n">Null</span> <span class="n">count</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="o">*</span> <span class="n">Validity</span> <span class="n">bitmap</span> <span class="n">buffer</span><span class="p">:</span> |
| |
| <span class="o">|</span> <span class="n">Byte</span> <span class="mi">0</span> <span class="p">(</span><span class="n">validity</span> <span class="n">bitmap</span><span class="p">)</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">1</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|--------------------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mi">00000101</span> <span class="o">|</span> <span class="mi">0</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| |
| <span class="o">*</span> <span class="n">Values</span> <span class="n">buffer</span> |
| |
| <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">0</span><span class="o">-</span><span class="mi">3</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">4</span><span class="o">-</span><span class="mi">7</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">8</span><span class="o">-</span><span class="mi">11</span> <span class="o">|</span> <span class="n">Bytes</span> <span class="mi">12</span><span class="o">-</span><span class="mi">63</span> <span class="o">|</span> |
| <span class="o">|-------------|-------------|-------------|-----------------------|</span> |
| <span class="o">|</span> <span class="mf">1.0</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="o">|</span> <span class="mf">2.0</span> <span class="o">|</span> <span class="n">unspecified</span> <span class="p">(</span><span class="n">padding</span><span class="p">)</span> <span class="o">|</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="buffer-listing-for-each-layout"> |
| <h3>Buffer Listing for Each Layout<a class="headerlink" href="#buffer-listing-for-each-layout" title="Permalink to this heading">#</a></h3> |
| <p>For the avoidance of ambiguity, we provide listing the order and type |
| of memory buffers for each layout.</p> |
| <table class="table" id="id7"> |
| <caption><span class="caption-text">Buffer Layouts</span><a class="headerlink" href="#id7" title="Permalink to this table">#</a></caption> |
| <colgroup> |
| <col style="width: 27.3%" /> |
| <col style="width: 18.2%" /> |
| <col style="width: 18.2%" /> |
| <col style="width: 18.2%" /> |
| <col style="width: 18.2%" /> |
| </colgroup> |
| <thead> |
| <tr class="row-odd"><th class="head"><p>Layout Type</p></th> |
| <th class="head"><p>Buffer 0</p></th> |
| <th class="head"><p>Buffer 1</p></th> |
| <th class="head"><p>Buffer 2</p></th> |
| <th class="head"><p>Variadic Buffers</p></th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="row-even"><td><p>Primitive</p></td> |
| <td><p>validity</p></td> |
| <td><p>data</p></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Variable Binary</p></td> |
| <td><p>validity</p></td> |
| <td><p>offsets</p></td> |
| <td><p>data</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Variable Binary View</p></td> |
| <td><p>validity</p></td> |
| <td><p>views</p></td> |
| <td></td> |
| <td><p>data</p></td> |
| </tr> |
| <tr class="row-odd"><td><p>List</p></td> |
| <td><p>validity</p></td> |
| <td><p>offsets</p></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Fixed-size List</p></td> |
| <td><p>validity</p></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Struct</p></td> |
| <td><p>validity</p></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Sparse Union</p></td> |
| <td><p>type ids</p></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Dense Union</p></td> |
| <td><p>type ids</p></td> |
| <td><p>offsets</p></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Null</p></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p>Dictionary-encoded</p></td> |
| <td><p>validity</p></td> |
| <td><p>data (indices)</p></td> |
| <td></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p>Run-end encoded</p></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| <td></td> |
| </tr> |
| </tbody> |
| </table> |
| </section> |
| </section> |
| <section id="logical-types"> |
| <h2>Logical Types<a class="headerlink" href="#logical-types" title="Permalink to this heading">#</a></h2> |
| <p>The <a class="reference external" href="https://github.com/apache/arrow/blob/main/format/Schema.fbs">Schema.fbs</a> defines built-in logical types supported by the |
| Arrow columnar format. Each logical type uses one of the above |
| physical layouts. Nested logical types may have different physical |
| layouts depending on the particular realization of the type.</p> |
| <p>We do not go into detail about the logical types definitions in this |
| document as we consider <a class="reference external" href="https://github.com/apache/arrow/blob/main/format/Schema.fbs">Schema.fbs</a> to be authoritative.</p> |
| </section> |
| <section id="serialization-and-interprocess-communication-ipc"> |
| <span id="format-ipc"></span><h2>Serialization and Interprocess Communication (IPC)<a class="headerlink" href="#serialization-and-interprocess-communication-ipc" title="Permalink to this heading">#</a></h2> |
| <p>The primitive unit of serialized data in the columnar format is the |
| “record batch”. Semantically, a record batch is an ordered collection |
| of arrays, known as its <strong>fields</strong>, each having the same length as one |
| another but potentially different data types. A record batch’s field |
| names and types collectively form the batch’s <strong>schema</strong>.</p> |
| <p>In this section we define a protocol for serializing record batches |
| into a stream of binary payloads and reconstructing record batches |
| from these payloads without need for memory copying.</p> |
| <p>The columnar IPC protocol utilizes a one-way stream of binary messages |
| of these types:</p> |
| <ul class="simple"> |
| <li><p>Schema</p></li> |
| <li><p>RecordBatch</p></li> |
| <li><p>DictionaryBatch</p></li> |
| </ul> |
| <p>We specify a so-called <em>encapsulated IPC message</em> format which |
| includes a serialized Flatbuffer type along with an optional message |
| body. We define this message format before describing how to serialize |
| each constituent IPC message type.</p> |
| <section id="encapsulated-message-format"> |
| <h3>Encapsulated message format<a class="headerlink" href="#encapsulated-message-format" title="Permalink to this heading">#</a></h3> |
| <p>For simple streaming and file-based serialization, we define a |
| “encapsulated” message format for interprocess communication. Such |
| messages can be “deserialized” into in-memory Arrow array objects by |
| examining only the message metadata without any need to copy or move |
| any of the actual data.</p> |
| <p>The encapsulated binary message format is as follows:</p> |
| <ul class="simple"> |
| <li><p>A 32-bit continuation indicator. The value <code class="docutils literal notranslate"><span class="pre">0xFFFFFFFF</span></code> indicates |
| a valid message. This component was introduced in version 0.15.0 in |
| part to address the 8-byte alignment requirement of Flatbuffers</p></li> |
| <li><p>A 32-bit little-endian length prefix indicating the metadata size</p></li> |
| <li><p>The message metadata as using the <code class="docutils literal notranslate"><span class="pre">Message</span></code> type defined in |
| <a class="reference external" href="https://github.com/apache/arrow/blob/main/format/Message.fbs">Message.fbs</a></p></li> |
| <li><p>Padding bytes to an 8-byte boundary</p></li> |
| <li><p>The message body, whose length must be a multiple of 8 bytes</p></li> |
| </ul> |
| <p>Schematically, we have:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">continuation</span><span class="p">:</span> <span class="mh">0xFFFFFFFF</span><span class="o">></span> |
| <span class="o"><</span><span class="n">metadata_size</span><span class="p">:</span> <span class="n">int32</span><span class="o">></span> |
| <span class="o"><</span><span class="n">metadata_flatbuffer</span><span class="p">:</span> <span class="nb">bytes</span><span class="o">></span> |
| <span class="o"><</span><span class="n">padding</span><span class="o">></span> |
| <span class="o"><</span><span class="n">message</span> <span class="n">body</span><span class="o">></span> |
| </pre></div> |
| </div> |
| <p>The complete serialized message must be a multiple of 8 bytes so that messages |
| can be relocated between streams. Otherwise the amount of padding between the |
| metadata and the message body could be non-deterministic.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">metadata_size</span></code> includes the size of the <code class="docutils literal notranslate"><span class="pre">Message</span></code> plus |
| padding. The <code class="docutils literal notranslate"><span class="pre">metadata_flatbuffer</span></code> contains a serialized <code class="docutils literal notranslate"><span class="pre">Message</span></code> |
| Flatbuffer value, which internally includes:</p> |
| <ul class="simple"> |
| <li><p>A version number</p></li> |
| <li><p>A particular message value (one of <code class="docutils literal notranslate"><span class="pre">Schema</span></code>, <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>, or |
| <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code>)</p></li> |
| <li><p>The size of the message body</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> field for any application-supplied metadata</p></li> |
| </ul> |
| <p>When read from an input stream, generally the <code class="docutils literal notranslate"><span class="pre">Message</span></code> metadata is |
| initially parsed and validated to obtain the body size. Then the body |
| can be read.</p> |
| </section> |
| <section id="schema-message"> |
| <h3>Schema message<a class="headerlink" href="#schema-message" title="Permalink to this heading">#</a></h3> |
| <p>The Flatbuffers files <a class="reference external" href="https://github.com/apache/arrow/blob/main/format/Schema.fbs">Schema.fbs</a> contains the definitions for all |
| built-in logical data types and the <code class="docutils literal notranslate"><span class="pre">Schema</span></code> metadata type which |
| represents the schema of a given record batch. A schema consists of |
| an ordered sequence of fields, each having a name and type. A |
| serialized <code class="docutils literal notranslate"><span class="pre">Schema</span></code> does not contain any data buffers, only type |
| metadata.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">Field</span></code> Flatbuffers type contains the metadata for a single |
| array. This includes:</p> |
| <ul class="simple"> |
| <li><p>The field’s name</p></li> |
| <li><p>The field’s logical type</p></li> |
| <li><p>Whether the field is semantically nullable. While this has no |
| bearing on the array’s physical layout, many systems distinguish |
| nullable and non-nullable fields and we want to allow them to |
| preserve this metadata to enable faithful schema round trips.</p></li> |
| <li><p>A collection of child <code class="docutils literal notranslate"><span class="pre">Field</span></code> values, for nested types</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">dictionary</span></code> property indicating whether the field is |
| dictionary-encoded or not. If it is, a dictionary “id” is assigned |
| to allow matching a subsequent dictionary IPC message with the |
| appropriate field.</p></li> |
| </ul> |
| <p>We additionally provide both schema-level and field-level |
| <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> attributes allowing for systems to insert their |
| own application defined metadata to customize behavior.</p> |
| </section> |
| <section id="recordbatch-message"> |
| <h3>RecordBatch message<a class="headerlink" href="#recordbatch-message" title="Permalink to this heading">#</a></h3> |
| <p>A RecordBatch message contains the actual data buffers corresponding |
| to the physical memory layout determined by a schema. The metadata for |
| this message provides the location and size of each buffer, permitting |
| Array data structures to be reconstructed using pointer arithmetic and |
| thus no memory copying.</p> |
| <p>The serialized form of the record batch is the following:</p> |
| <ul class="simple"> |
| <li><p>The <code class="docutils literal notranslate"><span class="pre">data</span> <span class="pre">header</span></code>, defined as the <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> type in |
| <a class="reference external" href="https://github.com/apache/arrow/blob/main/format/Message.fbs">Message.fbs</a>.</p></li> |
| <li><p>The <code class="docutils literal notranslate"><span class="pre">body</span></code>, a flat sequence of memory buffers written end-to-end |
| with appropriate padding to ensure a minimum of 8-byte alignment</p></li> |
| </ul> |
| <p>The data header contains the following:</p> |
| <ul class="simple"> |
| <li><p>The length and null count for each flattened field in the record |
| batch</p></li> |
| <li><p>The memory offset and length of each constituent <code class="docutils literal notranslate"><span class="pre">Buffer</span></code> in the |
| record batch’s body</p></li> |
| </ul> |
| <p>Fields and buffers are flattened by a pre-order depth-first traversal |
| of the fields in the record batch. For example, let’s consider the |
| schema</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">col1</span><span class="p">:</span> <span class="n">Struct</span><span class="o"><</span><span class="n">a</span><span class="p">:</span> <span class="n">Int32</span><span class="p">,</span> <span class="n">b</span><span class="p">:</span> <span class="n">List</span><span class="o"><</span><span class="n">item</span><span class="p">:</span> <span class="n">Int64</span><span class="o">></span><span class="p">,</span> <span class="n">c</span><span class="p">:</span> <span class="n">Float64</span><span class="o">></span> |
| <span class="n">col2</span><span class="p">:</span> <span class="n">Utf8</span> |
| </pre></div> |
| </div> |
| <p>The flattened version of this is:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">FieldNode</span> <span class="mi">0</span><span class="p">:</span> <span class="n">Struct</span> <span class="n">name</span><span class="o">=</span><span class="s1">'col1'</span> |
| <span class="n">FieldNode</span> <span class="mi">1</span><span class="p">:</span> <span class="n">Int32</span> <span class="n">name</span><span class="o">=</span><span class="s1">'a'</span> |
| <span class="n">FieldNode</span> <span class="mi">2</span><span class="p">:</span> <span class="n">List</span> <span class="n">name</span><span class="o">=</span><span class="s1">'b'</span> |
| <span class="n">FieldNode</span> <span class="mi">3</span><span class="p">:</span> <span class="n">Int64</span> <span class="n">name</span><span class="o">=</span><span class="s1">'item'</span> |
| <span class="n">FieldNode</span> <span class="mi">4</span><span class="p">:</span> <span class="n">Float64</span> <span class="n">name</span><span class="o">=</span><span class="s1">'c'</span> |
| <span class="n">FieldNode</span> <span class="mi">5</span><span class="p">:</span> <span class="n">Utf8</span> <span class="n">name</span><span class="o">=</span><span class="s1">'col2'</span> |
| </pre></div> |
| </div> |
| <p>For the buffers produced, we would have the following (refer to the |
| table above):</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">buffer</span> <span class="mi">0</span><span class="p">:</span> <span class="n">field</span> <span class="mi">0</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">1</span><span class="p">:</span> <span class="n">field</span> <span class="mi">1</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">2</span><span class="p">:</span> <span class="n">field</span> <span class="mi">1</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">3</span><span class="p">:</span> <span class="n">field</span> <span class="mi">2</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">4</span><span class="p">:</span> <span class="n">field</span> <span class="mi">2</span> <span class="n">offsets</span> |
| <span class="n">buffer</span> <span class="mi">5</span><span class="p">:</span> <span class="n">field</span> <span class="mi">3</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">6</span><span class="p">:</span> <span class="n">field</span> <span class="mi">3</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">7</span><span class="p">:</span> <span class="n">field</span> <span class="mi">4</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">8</span><span class="p">:</span> <span class="n">field</span> <span class="mi">4</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">9</span><span class="p">:</span> <span class="n">field</span> <span class="mi">5</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">10</span><span class="p">:</span> <span class="n">field</span> <span class="mi">5</span> <span class="n">offsets</span> |
| <span class="n">buffer</span> <span class="mi">11</span><span class="p">:</span> <span class="n">field</span> <span class="mi">5</span> <span class="n">data</span> |
| </pre></div> |
| </div> |
| <p>The <code class="docutils literal notranslate"><span class="pre">Buffer</span></code> Flatbuffers value describes the location and size of a |
| piece of memory. Generally these are interpreted relative to the |
| <strong>encapsulated message format</strong> defined below.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">size</span></code> field of <code class="docutils literal notranslate"><span class="pre">Buffer</span></code> is not required to account for padding |
| bytes. Since this metadata can be used to communicate in-memory pointer |
| addresses between libraries, it is recommended to set <code class="docutils literal notranslate"><span class="pre">size</span></code> to the actual |
| memory size rather than the padded size.</p> |
| </section> |
| <section id="variadic-buffers"> |
| <span id="id6"></span><h3>Variadic buffers<a class="headerlink" href="#variadic-buffers" title="Permalink to this heading">#</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version Arrow: </span>Columnar Format 1.4</p> |
| </div> |
| <p>Some types such as Utf8View are represented using a variable number of buffers. |
| For each such Field in the pre-ordered flattened logical schema, there will be |
| an entry in <code class="docutils literal notranslate"><span class="pre">variadicBufferCounts</span></code> to indicate the number of variadic buffers |
| which belong to that Field in the current RecordBatch.</p> |
| <p>For example, consider the schema</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">col1</span><span class="p">:</span> <span class="n">Struct</span><span class="o"><</span><span class="n">a</span><span class="p">:</span> <span class="n">Int32</span><span class="p">,</span> <span class="n">b</span><span class="p">:</span> <span class="n">BinaryView</span><span class="p">,</span> <span class="n">c</span><span class="p">:</span> <span class="n">Float64</span><span class="o">></span> |
| <span class="n">col2</span><span class="p">:</span> <span class="n">Utf8View</span> |
| </pre></div> |
| </div> |
| <p>This has two fields with variadic buffers, so <code class="docutils literal notranslate"><span class="pre">variadicBufferCounts</span></code> will |
| have two entries in each RecordBatch. For a RecordBatch of this schema with |
| <code class="docutils literal notranslate"><span class="pre">variadicBufferCounts</span> <span class="pre">=</span> <span class="pre">[3,</span> <span class="pre">2]</span></code>, the flattened buffers would be:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">buffer</span> <span class="mi">0</span><span class="p">:</span> <span class="n">col1</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">1</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">a</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">2</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">a</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">3</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">b</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">4</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">b</span> <span class="n">views</span> |
| <span class="n">buffer</span> <span class="mi">5</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">b</span> <span class="n">data</span> |
| <span class="n">buffer</span> <span class="mi">6</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">b</span> <span class="n">data</span> |
| <span class="n">buffer</span> <span class="mi">7</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">b</span> <span class="n">data</span> |
| <span class="n">buffer</span> <span class="mi">8</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">c</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">9</span><span class="p">:</span> <span class="n">col1</span><span class="o">.</span><span class="n">c</span> <span class="n">values</span> |
| <span class="n">buffer</span> <span class="mi">10</span><span class="p">:</span> <span class="n">col2</span> <span class="n">validity</span> |
| <span class="n">buffer</span> <span class="mi">11</span><span class="p">:</span> <span class="n">col2</span> <span class="n">views</span> |
| <span class="n">buffer</span> <span class="mi">12</span><span class="p">:</span> <span class="n">col2</span> <span class="n">data</span> |
| <span class="n">buffer</span> <span class="mi">13</span><span class="p">:</span> <span class="n">col2</span> <span class="n">data</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="byte-order-endianness"> |
| <h3>Byte Order (<a class="reference external" href="https://en.wikipedia.org/wiki/Endianness">Endianness</a>)<a class="headerlink" href="#byte-order-endianness" title="Permalink to this heading">#</a></h3> |
| <p>The Arrow format is little endian by default.</p> |
| <p>Serialized Schema metadata has an endianness field indicating |
| endianness of RecordBatches. Typically this is the endianness of the |
| system where the RecordBatch was generated. The main use case is |
| exchanging RecordBatches between systems with the same Endianness. At |
| first we will return an error when trying to read a Schema with an |
| endianness that does not match the underlying system. The reference |
| implementation is focused on Little Endian and provides tests for |
| it. Eventually we may provide automatic conversion via byte swapping.</p> |
| </section> |
| <section id="ipc-streaming-format"> |
| <h3>IPC Streaming Format<a class="headerlink" href="#ipc-streaming-format" title="Permalink to this heading">#</a></h3> |
| <p>We provide a streaming protocol or “format” for record batches. It is |
| presented as a sequence of encapsulated messages, each of which |
| follows the format above. The schema comes first in the stream, and it |
| is the same for all of the record batches that follow. If any fields |
| in the schema are dictionary-encoded, one or more <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code> |
| messages will be included. <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code> and <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> |
| messages may be interleaved, but before any dictionary key is used in |
| a <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code> it should be defined in a <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code>.</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">SCHEMA</span><span class="o">></span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="n">k</span> <span class="o">-</span> <span class="mi">1</span><span class="o">></span> |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">0</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="n">x</span> <span class="n">DELTA</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="n">y</span> <span class="n">DELTA</span><span class="o">></span> |
| <span class="o">...</span> |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="o">></span> |
| <span class="o"><</span><span class="n">EOS</span> <span class="p">[</span><span class="n">optional</span><span class="p">]:</span> <span class="mh">0xFFFFFFFF</span> <span class="mh">0x00000000</span><span class="o">></span> |
| </pre></div> |
| </div> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>An edge-case for interleaved dictionary and record batches occurs |
| when the record batches contain dictionary encoded arrays that are |
| completely null. In this case, the dictionary for the encoded column might |
| appear after the first record batch.</p> |
| </div> |
| <p>When a stream reader implementation is reading a stream, after each |
| message, it may read the next 8 bytes to determine both if the stream |
| continues and the size of the message metadata that follows. Once the |
| message flatbuffer is read, you can then read the message body.</p> |
| <p>The stream writer can signal end-of-stream (EOS) either by writing 8 bytes |
| containing the 4-byte continuation indicator (<code class="docutils literal notranslate"><span class="pre">0xFFFFFFFF</span></code>) followed by 0 |
| metadata length (<code class="docutils literal notranslate"><span class="pre">0x00000000</span></code>) or closing the stream interface. We |
| recommend the “.arrows” file extension for the streaming format although |
| in many cases these streams will not ever be stored as files.</p> |
| </section> |
| <section id="ipc-file-format"> |
| <h3>IPC File Format<a class="headerlink" href="#ipc-file-format" title="Permalink to this heading">#</a></h3> |
| <p>We define a “file format” supporting random access that is an extension of |
| the stream format. The file starts and ends with a magic string <code class="docutils literal notranslate"><span class="pre">ARROW1</span></code> |
| (plus padding). What follows in the file is identical to the stream format. |
| At the end of the file, we write a <em>footer</em> containing a redundant copy of |
| the schema (which is a part of the streaming format) plus memory offsets and |
| sizes for each of the data blocks in the file. This enables random access to |
| any record batch in the file. See <a class="reference external" href="https://github.com/apache/arrow/blob/main/format/File.fbs">File.fbs</a> for the precise details of the |
| file footer.</p> |
| <p>Schematically we have:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">magic</span> <span class="n">number</span> <span class="s2">"ARROW1"</span><span class="o">></span> |
| <span class="o"><</span><span class="n">empty</span> <span class="n">padding</span> <span class="nb">bytes</span> <span class="p">[</span><span class="n">to</span> <span class="mi">8</span> <span class="n">byte</span> <span class="n">boundary</span><span class="p">]</span><span class="o">></span> |
| <span class="o"><</span><span class="n">STREAMING</span> <span class="n">FORMAT</span> <span class="k">with</span> <span class="n">EOS</span><span class="o">></span> |
| <span class="o"><</span><span class="n">FOOTER</span><span class="o">></span> |
| <span class="o"><</span><span class="n">FOOTER</span> <span class="n">SIZE</span><span class="p">:</span> <span class="n">int32</span><span class="o">></span> |
| <span class="o"><</span><span class="n">magic</span> <span class="n">number</span> <span class="s2">"ARROW1"</span><span class="o">></span> |
| </pre></div> |
| </div> |
| <p>In the file format, there is no requirement that dictionary keys |
| should be defined in a <code class="docutils literal notranslate"><span class="pre">DictionaryBatch</span></code> before they are used in a |
| <code class="docutils literal notranslate"><span class="pre">RecordBatch</span></code>, as long as the keys are defined somewhere in the |
| file. Further more, it is invalid to have more than one <strong>non-delta</strong> |
| dictionary batch per dictionary ID (i.e. dictionary replacement is not |
| supported). Delta dictionaries are applied in the order they appear in |
| the file footer. We recommend the “.arrow” extension for files created with |
| this format. Note that files created with this format are sometimes called |
| “Feather V2” or with the “.feather” extension, the name and the extension |
| derived from “Feather (V1)”, which was a proof of concept early in |
| the Arrow project for language-agnostic fast data frame storage for |
| Python (pandas) and R.</p> |
| </section> |
| <section id="dictionary-messages"> |
| <h3>Dictionary Messages<a class="headerlink" href="#dictionary-messages" title="Permalink to this heading">#</a></h3> |
| <p>Dictionaries are written in the stream and file formats as a sequence of record |
| batches, each having a single field. The complete semantic schema for a |
| sequence of record batches, therefore, consists of the schema along with all of |
| the dictionaries. The dictionary types are found in the schema, so it is |
| necessary to read the schema to first determine the dictionary types so that |
| the dictionaries can be properly interpreted:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">table</span> <span class="n">DictionaryBatch</span> <span class="p">{</span> |
| <span class="nb">id</span><span class="p">:</span> <span class="n">long</span><span class="p">;</span> |
| <span class="n">data</span><span class="p">:</span> <span class="n">RecordBatch</span><span class="p">;</span> |
| <span class="n">isDelta</span><span class="p">:</span> <span class="n">boolean</span> <span class="o">=</span> <span class="n">false</span><span class="p">;</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>The dictionary <code class="docutils literal notranslate"><span class="pre">id</span></code> in the message metadata can be referenced one or more times |
| in the schema, so that dictionaries can even be used for multiple fields. See |
| the <a class="reference internal" href="#dictionary-encoded-layout"><span class="std std-ref">Dictionary-encoded Layout</span></a> section for more about the semantics of |
| dictionary-encoded data.</p> |
| <p>The dictionary <code class="docutils literal notranslate"><span class="pre">isDelta</span></code> flag allows existing dictionaries to be |
| expanded for future record batch materializations. A dictionary batch |
| with <code class="docutils literal notranslate"><span class="pre">isDelta</span></code> set indicates that its vector should be concatenated |
| with those of any previous batches with the same <code class="docutils literal notranslate"><span class="pre">id</span></code>. In a stream |
| which encodes one column, the list of strings <code class="docutils literal notranslate"><span class="pre">["A",</span> <span class="pre">"B",</span> <span class="pre">"C",</span> <span class="pre">"B",</span> |
| <span class="pre">"D",</span> <span class="pre">"C",</span> <span class="pre">"E",</span> <span class="pre">"A"]</span></code>, with a delta dictionary batch could take the |
| form:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">SCHEMA</span><span class="o">></span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="s2">"A"</span> |
| <span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="s2">"B"</span> |
| <span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="s2">"C"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">0</span><span class="o">></span> |
| <span class="mi">0</span> |
| <span class="mi">1</span> |
| <span class="mi">2</span> |
| <span class="mi">1</span> |
| |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span> <span class="n">DELTA</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">3</span><span class="p">)</span> <span class="s2">"D"</span> |
| <span class="p">(</span><span class="mi">4</span><span class="p">)</span> <span class="s2">"E"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">1</span><span class="o">></span> |
| <span class="mi">3</span> |
| <span class="mi">2</span> |
| <span class="mi">4</span> |
| <span class="mi">0</span> |
| <span class="n">EOS</span> |
| </pre></div> |
| </div> |
| <p>Alternatively, if <code class="docutils literal notranslate"><span class="pre">isDelta</span></code> is set to false, then the dictionary |
| replaces the existing dictionary for the same ID. Using the same |
| example as above, an alternate encoding could be:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o"><</span><span class="n">SCHEMA</span><span class="o">></span> |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="s2">"A"</span> |
| <span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="s2">"B"</span> |
| <span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="s2">"C"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">0</span><span class="o">></span> |
| <span class="mi">0</span> |
| <span class="mi">1</span> |
| <span class="mi">2</span> |
| <span class="mi">1</span> |
| |
| <span class="o"><</span><span class="n">DICTIONARY</span> <span class="mi">0</span><span class="o">></span> |
| <span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="s2">"A"</span> |
| <span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="s2">"C"</span> |
| <span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="s2">"D"</span> |
| <span class="p">(</span><span class="mi">3</span><span class="p">)</span> <span class="s2">"E"</span> |
| |
| <span class="o"><</span><span class="n">RECORD</span> <span class="n">BATCH</span> <span class="mi">1</span><span class="o">></span> |
| <span class="mi">2</span> |
| <span class="mi">1</span> |
| <span class="mi">3</span> |
| <span class="mi">0</span> |
| <span class="n">EOS</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="custom-application-metadata"> |
| <h3>Custom Application Metadata<a class="headerlink" href="#custom-application-metadata" title="Permalink to this heading">#</a></h3> |
| <p>We provide a <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> field at three levels to provide a |
| mechanism for developers to pass application-specific metadata in |
| Arrow protocol messages. This includes <code class="docutils literal notranslate"><span class="pre">Field</span></code>, <code class="docutils literal notranslate"><span class="pre">Schema</span></code>, and |
| <code class="docutils literal notranslate"><span class="pre">Message</span></code>.</p> |
| <p>The colon symbol <code class="docutils literal notranslate"><span class="pre">:</span></code> is to be used as a namespace separator. It can |
| be used multiple times in a key.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">ARROW</span></code> pattern is a reserved namespace for internal Arrow use |
| in the <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> fields. For example, |
| <code class="docutils literal notranslate"><span class="pre">ARROW:extension:name</span></code>.</p> |
| </section> |
| <section id="extension-types"> |
| <span id="format-metadata-extension-types"></span><h3>Extension Types<a class="headerlink" href="#extension-types" title="Permalink to this heading">#</a></h3> |
| <p>User-defined “extension” types can be defined setting certain |
| <code class="docutils literal notranslate"><span class="pre">KeyValue</span></code> pairs in <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> in the <code class="docutils literal notranslate"><span class="pre">Field</span></code> metadata |
| structure. These extension keys are:</p> |
| <ul class="simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">'ARROW:extension:name'</span></code> for the string name identifying the |
| custom data type. We recommend that you use a “namespace”-style |
| prefix for extension type names to minimize the possibility of |
| conflicts with multiple Arrow readers and writers in the same |
| application. For example, use <code class="docutils literal notranslate"><span class="pre">myorg.name_of_type</span></code> instead of |
| simply <code class="docutils literal notranslate"><span class="pre">name_of_type</span></code></p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">'ARROW:extension:metadata'</span></code> for a serialized representation |
| of the <code class="docutils literal notranslate"><span class="pre">ExtensionType</span></code> necessary to reconstruct the custom type</p></li> |
| </ul> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Extension names beginning with <code class="docutils literal notranslate"><span class="pre">arrow.</span></code> are reserved for |
| <a class="reference internal" href="CanonicalExtensions.html#format-canonical-extensions"><span class="std std-ref">canonical extension types</span></a>, |
| they should not be used for third-party extension types.</p> |
| </div> |
| <p>This extension metadata can annotate any of the built-in Arrow logical |
| types. The intent is that an implementation that does not support an |
| extension type can still handle the underlying data. For example a |
| 16-byte UUID value could be embedded in <code class="docutils literal notranslate"><span class="pre">FixedSizeBinary(16)</span></code>, and |
| implementations that do not have this extension type can still work |
| with the underlying binary values and pass along the |
| <code class="docutils literal notranslate"><span class="pre">custom_metadata</span></code> in subsequent Arrow protocol messages.</p> |
| <p>Extension types may or may not use the |
| <code class="docutils literal notranslate"><span class="pre">'ARROW:extension:metadata'</span></code> field. Let’s consider some example |
| extension types:</p> |
| <ul class="simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">uuid</span></code> represented as <code class="docutils literal notranslate"><span class="pre">FixedSizeBinary(16)</span></code> with empty metadata</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">latitude-longitude</span></code> represented as <code class="docutils literal notranslate"><span class="pre">struct<latitude:</span> <span class="pre">double,</span> |
| <span class="pre">longitude:</span> <span class="pre">double></span></code>, and empty metadata</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">tensor</span></code> (multidimensional array) stored as <code class="docutils literal notranslate"><span class="pre">Binary</span></code> values and |
| having serialized metadata indicating the data type and shape of |
| each value. This could be JSON like <code class="docutils literal notranslate"><span class="pre">{'type':</span> <span class="pre">'int8',</span> <span class="pre">'shape':</span> <span class="pre">[4,</span> |
| <span class="pre">5]}</span></code> for a 4x5 cell tensor.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">trading-time</span></code> represented as <code class="docutils literal notranslate"><span class="pre">Timestamp</span></code> with serialized |
| metadata indicating the market trading calendar the data corresponds |
| to</p></li> |
| </ul> |
| <div class="admonition seealso"> |
| <p class="admonition-title">See also</p> |
| <p><a class="reference internal" href="CanonicalExtensions.html#format-canonical-extensions"><span class="std std-ref">Canonical Extension Types</span></a></p> |
| </div> |
| </section> |
| </section> |
| <section id="implementation-guidelines"> |
| <h2>Implementation guidelines<a class="headerlink" href="#implementation-guidelines" title="Permalink to this heading">#</a></h2> |
| <p>An execution engine (or framework, or UDF executor, or storage engine, |
| etc) can implement only a subset of the Arrow spec and/or extend it |
| given the following constraints:</p> |
| <section id="implementing-a-subset-of-the-spec"> |
| <h3>Implementing a subset of the spec<a class="headerlink" href="#implementing-a-subset-of-the-spec" title="Permalink to this heading">#</a></h3> |
| <ul class="simple"> |
| <li><p><strong>If only producing (and not consuming) arrow vectors</strong>: Any subset |
| of the vector spec and the corresponding metadata can be implemented.</p></li> |
| <li><p><strong>If consuming and producing vectors</strong>: There is a minimal subset of |
| vectors to be supported. Production of a subset of vectors and |
| their corresponding metadata is always fine. Consumption of vectors |
| should at least convert the unsupported input vectors to the |
| supported subset (for example Timestamp.millis to timestamp.micros |
| or int32 to int64).</p></li> |
| </ul> |
| </section> |
| <section id="extensibility"> |
| <h3>Extensibility<a class="headerlink" href="#extensibility" title="Permalink to this heading">#</a></h3> |
| <p>An execution engine implementor can also extend their memory |
| representation with their own vectors internally as long as they are |
| never exposed. Before sending data to another system expecting Arrow |
| data, these custom vectors should be converted to a type that exist in |
| the Arrow spec.</p> |
| </section> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="Versioning.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Format Versioning and Stability</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="CanonicalExtensions.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Canonical Extension Types</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#terminology">Terminology</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#physical-memory-layout">Physical Memory Layout</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#buffer-alignment-and-padding">Buffer Alignment and Padding</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#array-lengths">Array lengths</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#null-count">Null count</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#validity-bitmaps">Validity bitmaps</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#fixed-size-primitive-layout">Fixed-size Primitive Layout</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#variable-size-binary-layout">Variable-size Binary Layout</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#variable-size-binary-view-layout">Variable-size Binary View Layout</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#variable-size-list-layout">Variable-size List Layout</a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#list-layout">List Layout</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#listview-layout">ListView Layout</a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#fixed-size-list-layout">Fixed-Size List Layout</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#struct-layout">Struct Layout</a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#struct-validity">Struct Validity</a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#union-layout">Union Layout</a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#dense-union">Dense Union</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#sparse-union">Sparse Union</a></li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#null-layout">Null Layout</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#dictionary-encoded-layout">Dictionary-encoded Layout</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#run-end-encoded-layout">Run-End Encoded Layout</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#buffer-listing-for-each-layout">Buffer Listing for Each Layout</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#logical-types">Logical Types</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#serialization-and-interprocess-communication-ipc">Serialization and Interprocess Communication (IPC)</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#encapsulated-message-format">Encapsulated message format</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#schema-message">Schema message</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#recordbatch-message">RecordBatch message</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#variadic-buffers">Variadic buffers</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#byte-order-endianness">Byte Order (Endianness)</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#ipc-streaming-format">IPC Streaming Format</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#ipc-file-format">IPC File Format</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#dictionary-messages">Dictionary Messages</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#custom-application-metadata">Custom Application Metadata</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#extension-types">Extension Types</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#implementation-guidelines">Implementation guidelines</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#implementing-a-subset-of-the-spec">Implementing a subset of the spec</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#extensibility">Extensibility</a></li> |
| </ul> |
| </li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/format/Columnar.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script> |
| <script src="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2024 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
