| |
| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" /> |
| |
| <title>Canonical Extension Types — Apache Arrow v17.0.0.dev77</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; |
| </script> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| |
| |
| <link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> |
| <script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script> |
| <script src="../_static/doctools.js"></script> |
| <script src="../_static/sphinx_highlight.js"></script> |
| <script src="../_static/clipboard.min.js"></script> |
| <script src="../_static/copybutton.js"></script> |
| <script src="../_static/design-tabs.js"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'format/CanonicalExtensions';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.15.2'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/format/CanonicalExtensions.html" /> |
| <link rel="icon" href="../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Other Data Structures" href="Other.html" /> |
| <link rel="prev" title="Arrow Columnar Format" href="Columnar.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i> |
| Back to top |
| </button> |
| |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__primary" |
| id="__primary"/> |
| <label class="overlay overlay-primary" for="__primary"></label> |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__secondary" |
| id="__secondary"/> |
| <label class="overlay overlay-secondary" for="__secondary"></label> |
| |
| <div class="search-button__wrapper"> |
| <div class="search-button__overlay"></div> |
| <div class="search-button__search-container"> |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| id="search-input" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form></div> |
| </div> |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar"> |
| <div class="bd-header__inner bd-page-width"> |
| <label class="sidebar-toggle primary-toggle" for="__primary"> |
| <span class="fa-solid fa-bars"></span> |
| </label> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev77 - Home"/> |
| <script>document.write(`<img src="../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev77 - Home"/>`);</script> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| |
| <label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0"> |
| <span class="fa-solid fa-outdent"></span> |
| </label> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <div class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links-2" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="Versioning.html">Format Versioning and Stability</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Columnar.html">Arrow Columnar Format</a></li> |
| <li class="toctree-l1 current active"><a class="current reference internal" href="#">Canonical Extension Types</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Other.html">Other Data Structures</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="CDataInterface.html">The Arrow C data interface</a><input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="CDataInterface/PyCapsuleInterface.html">The Arrow PyCapsule Interface</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="CStreamInterface.html">The Arrow C stream interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="CDeviceDataInterface.html">The Arrow C Device data interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="FlightSql.html">Arrow Flight SQL</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="ADBC.html">ADBC: Arrow Database Connectivity</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/C.html">ADBC C API Specification</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/Go.html">ADBC Go API Specification</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/Java.html">ADBC Java API Specification</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="Changing.html">Changing the Apache Arrow Format Specification</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Integration.html">Integration Testing</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Glossary.html">Glossary</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| </div> |
| |
| <div id="rtd-footer-container"></div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| |
| |
| <nav aria-label="Breadcrumb"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Specifications</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page">Canonical...</li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="canonical-extension-types"> |
| <span id="format-canonical-extensions"></span><h1>Canonical Extension Types<a class="headerlink" href="#canonical-extension-types" title="Permalink to this heading">#</a></h1> |
| <section id="introduction"> |
| <h2>Introduction<a class="headerlink" href="#introduction" title="Permalink to this heading">#</a></h2> |
| <p>The Arrow columnar format allows defining |
| <a class="reference internal" href="Columnar.html#format-metadata-extension-types"><span class="std std-ref">extension types</span></a> so as to extend |
| standard Arrow data types with custom semantics. Often these semantics |
| will be specific to a system or application. However, it is beneficial |
| to share the definitions of well-known extension types so as to improve |
| interoperability between different systems integrating Arrow columnar data.</p> |
| <section id="standardization"> |
| <h3>Standardization<a class="headerlink" href="#standardization" title="Permalink to this heading">#</a></h3> |
| <p>These rules must be followed for the standardization of canonical extension |
| types:</p> |
| <ul class="simple"> |
| <li><p>Canonical extension types are described and maintained below in this document.</p></li> |
| <li><p>Each canonical extension type requires a distinct discussion and vote |
| on the <a class="reference external" href="https://arrow.apache.org/community/">Arrow development mailing-list</a>.</p></li> |
| <li><p>The specification text to be added <em>must</em> follow these requirements:</p> |
| <ol class="arabic simple"> |
| <li><p>It <em>must</em> define a well-defined extension name starting with “<code class="docutils literal notranslate"><span class="pre">arrow.</span></code>”.</p></li> |
| <li><p>Its parameters, if any, <em>must</em> be described in the proposal.</p></li> |
| <li><p>Its serialization <em>must</em> be described in the proposal and should |
| not require unduly implementation work or unusual software dependencies |
| (for example, a trivial custom text format or JSON would be acceptable).</p></li> |
| <li><p>Its expected semantics <em>should</em> be described as well and any |
| potential ambiguities or pain points addressed or at least mentioned.</p></li> |
| </ol> |
| </li> |
| <li><p>The extension type <em>should</em> have one implementation submitted; |
| preferably two if non-trivial (for example if parameterized).</p></li> |
| </ul> |
| </section> |
| <section id="making-modifications"> |
| <h3>Making Modifications<a class="headerlink" href="#making-modifications" title="Permalink to this heading">#</a></h3> |
| <p>Like standard Arrow data types, canonical extension types should be considered |
| stable once standardized. Modifying a canonical extension type (for example |
| to expand the set of parameters) should be an exceptional event, follow the |
| same rules as laid out above, and provide backwards compatibility guarantees.</p> |
| </section> |
| </section> |
| <section id="official-list"> |
| <h2>Official List<a class="headerlink" href="#official-list" title="Permalink to this heading">#</a></h2> |
| <section id="fixed-shape-tensor"> |
| <span id="fixed-shape-tensor-extension"></span><h3>Fixed shape tensor<a class="headerlink" href="#fixed-shape-tensor" title="Permalink to this heading">#</a></h3> |
| <ul> |
| <li><p>Extension name: <cite>arrow.fixed_shape_tensor</cite>.</p></li> |
| <li><p>The storage type of the extension: <code class="docutils literal notranslate"><span class="pre">FixedSizeList</span></code> where:</p> |
| <ul class="simple"> |
| <li><p><strong>value_type</strong> is the data type of individual tensor elements.</p></li> |
| <li><p><strong>list_size</strong> is the product of all the elements in tensor shape.</p></li> |
| </ul> |
| </li> |
| <li><p>Extension type parameters:</p> |
| <ul class="simple"> |
| <li><p><strong>value_type</strong> = the Arrow data type of individual tensor elements.</p></li> |
| <li><p><strong>shape</strong> = the physical shape of the contained tensors |
| as an array.</p></li> |
| </ul> |
| <p>Optional parameters describing the logical layout:</p> |
| <ul> |
| <li><p><strong>dim_names</strong> = explicit names to tensor dimensions |
| as an array. The length of it should be equal to the shape |
| length and equal to the number of dimensions.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">dim_names</span></code> can be used if the dimensions have well-known |
| names and they map to the physical layout (row-major).</p> |
| </li> |
| <li><p><strong>permutation</strong> = indices of the desired ordering of the |
| original dimensions, defined as an array.</p> |
| <p>The indices contain a permutation of the values [0, 1, .., N-1] where |
| N is the number of dimensions. The permutation indicates which |
| dimension of the logical layout corresponds to which dimension of the |
| physical tensor (the i-th dimension of the logical view corresponds |
| to the dimension with number <code class="docutils literal notranslate"><span class="pre">permutations[i]</span></code> of the physical tensor).</p> |
| <p>Permutation can be useful in case the logical order of |
| the tensor is a permutation of the physical order (row-major).</p> |
| <p>When logical and physical layout are equal, the permutation will always |
| be ([0, 1, .., N-1]) and can therefore be left out.</p> |
| </li> |
| </ul> |
| </li> |
| <li><p>Description of the serialization:</p> |
| <p>The metadata must be a valid JSON object including shape of |
| the contained tensors as an array with key <strong>“shape”</strong> plus optional |
| dimension names with keys <strong>“dim_names”</strong> and ordering of the |
| dimensions with key <strong>“permutation”</strong>.</p> |
| <ul> |
| <li><p>Example: <code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">"shape":</span> <span class="pre">[2,</span> <span class="pre">5]}</span></code></p></li> |
| <li><p>Example with <code class="docutils literal notranslate"><span class="pre">dim_names</span></code> metadata for NCHW ordered data:</p> |
| <p><code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">"shape":</span> <span class="pre">[100,</span> <span class="pre">200,</span> <span class="pre">500],</span> <span class="pre">"dim_names":</span> <span class="pre">["C",</span> <span class="pre">"H",</span> <span class="pre">"W"]}</span></code></p> |
| </li> |
| <li><p>Example of permuted 3-dimensional tensor:</p> |
| <p><code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">"shape":</span> <span class="pre">[100,</span> <span class="pre">200,</span> <span class="pre">500],</span> <span class="pre">"permutation":</span> <span class="pre">[2,</span> <span class="pre">0,</span> <span class="pre">1]}</span></code></p> |
| <p>This is the physical layout shape and the shape of the logical |
| layout would in this case be <code class="docutils literal notranslate"><span class="pre">[500,</span> <span class="pre">100,</span> <span class="pre">200]</span></code>.</p> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Elements in a fixed shape tensor extension array are stored |
| in row-major/C-contiguous order.</p> |
| </div> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Other Data Structures in Arrow include a |
| <a class="reference external" href="https://arrow.apache.org/docs/format/Other.html">Tensor (Multi-dimensional Array)</a> |
| to be used as a message in the interprocess communication machinery (IPC).</p> |
| <p>This structure has no relationship with the Fixed shape tensor extension type defined |
| by this specification. Instead, this extension type lets one use fixed shape tensors |
| as elements in a field of a RecordBatch or a Table.</p> |
| </div> |
| </section> |
| <section id="variable-shape-tensor"> |
| <span id="variable-shape-tensor-extension"></span><h3>Variable shape tensor<a class="headerlink" href="#variable-shape-tensor" title="Permalink to this heading">#</a></h3> |
| <ul> |
| <li><p>Extension name: <cite>arrow.variable_shape_tensor</cite>.</p></li> |
| <li><p>The storage type of the extension is: <code class="docutils literal notranslate"><span class="pre">StructArray</span></code> where struct |
| is composed of <strong>data</strong> and <strong>shape</strong> fields describing a single |
| tensor per row:</p> |
| <ul class="simple"> |
| <li><p><strong>data</strong> is a <code class="docutils literal notranslate"><span class="pre">List</span></code> holding tensor elements (each list element is |
| a single tensor). The List’s value type is the value type of the tensor, |
| such as an integer or floating-point type.</p></li> |
| <li><p><strong>shape</strong> is a <code class="docutils literal notranslate"><span class="pre">FixedSizeList<int32>[ndim]</span></code> of the tensor shape where |
| the size of the list <code class="docutils literal notranslate"><span class="pre">ndim</span></code> is equal to the number of dimensions of the |
| tensor.</p></li> |
| </ul> |
| </li> |
| <li><p>Extension type parameters:</p> |
| <ul class="simple"> |
| <li><p><strong>value_type</strong> = the Arrow data type of individual tensor elements.</p></li> |
| </ul> |
| <p>Optional parameters describing the logical layout:</p> |
| <ul> |
| <li><p><strong>dim_names</strong> = explicit names to tensor dimensions |
| as an array. The length of it should be equal to the shape |
| length and equal to the number of dimensions.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">dim_names</span></code> can be used if the dimensions have well-known |
| names and they map to the physical layout (row-major).</p> |
| </li> |
| <li><p><strong>permutation</strong> = indices of the desired ordering of the |
| original dimensions, defined as an array.</p> |
| <p>The indices contain a permutation of the values [0, 1, .., N-1] where |
| N is the number of dimensions. The permutation indicates which |
| dimension of the logical layout corresponds to which dimension of the |
| physical tensor (the i-th dimension of the logical view corresponds |
| to the dimension with number <code class="docutils literal notranslate"><span class="pre">permutations[i]</span></code> of the physical tensor).</p> |
| <p>Permutation can be useful in case the logical order of |
| the tensor is a permutation of the physical order (row-major).</p> |
| <p>When logical and physical layout are equal, the permutation will always |
| be ([0, 1, .., N-1]) and can therefore be left out.</p> |
| </li> |
| <li><p><strong>uniform_shape</strong> = sizes of individual tensor’s dimensions which are |
| guaranteed to stay constant in uniform dimensions and can vary in |
| non-uniform dimensions. This holds over all tensors in the array. |
| Sizes in uniform dimensions are represented with int32 values, while |
| sizes of the non-uniform dimensions are not known in advance and are |
| represented with null. If <code class="docutils literal notranslate"><span class="pre">uniform_shape</span></code> is not provided it is assumed |
| that all dimensions are non-uniform. |
| An array containing a tensor with shape (2, 3, 4) and whose first and |
| last dimensions are uniform would have <code class="docutils literal notranslate"><span class="pre">uniform_shape</span></code> (2, null, 4). |
| This allows for interpreting the tensor correctly without accounting for |
| uniform dimensions while still permitting optional optimizations that |
| take advantage of the uniformity.</p></li> |
| </ul> |
| </li> |
| <li><p>Description of the serialization:</p> |
| <p>The metadata must be a valid JSON object that optionally includes |
| dimension names with keys <strong>“dim_names”</strong> and ordering of dimensions |
| with key <strong>“permutation”</strong>. |
| Shapes of tensors can be defined in a subset of dimensions by providing |
| key <strong>“uniform_shape”</strong>. |
| Minimal metadata is an empty string.</p> |
| <ul> |
| <li><p>Example with <code class="docutils literal notranslate"><span class="pre">dim_names</span></code> metadata for NCHW ordered data (note that the first |
| logical dimension, <code class="docutils literal notranslate"><span class="pre">N</span></code>, is mapped to the <strong>data</strong> List array: each element in the List |
| is a CHW tensor and the List of tensors implicitly constitutes a single NCHW tensor):</p> |
| <p><code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">"dim_names":</span> <span class="pre">["C",</span> <span class="pre">"H",</span> <span class="pre">"W"]</span> <span class="pre">}</span></code></p> |
| </li> |
| <li><p>Example with <code class="docutils literal notranslate"><span class="pre">uniform_shape</span></code> metadata for a set of color images |
| with fixed height, variable width and three color channels:</p> |
| <p><code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">"dim_names":</span> <span class="pre">["H",</span> <span class="pre">"W",</span> <span class="pre">"C"],</span> <span class="pre">"uniform_shape":</span> <span class="pre">[400,</span> <span class="pre">null,</span> <span class="pre">3]</span> <span class="pre">}</span></code></p> |
| </li> |
| <li><p>Example of permuted 3-dimensional tensor:</p> |
| <p><code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">"permutation":</span> <span class="pre">[2,</span> <span class="pre">0,</span> <span class="pre">1]</span> <span class="pre">}</span></code></p> |
| <p>For example, if the physical <strong>shape</strong> of an individual tensor |
| is <code class="docutils literal notranslate"><span class="pre">[100,</span> <span class="pre">200,</span> <span class="pre">500]</span></code>, this permutation would denote a logical shape |
| of <code class="docutils literal notranslate"><span class="pre">[500,</span> <span class="pre">100,</span> <span class="pre">200]</span></code>.</p> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>With the exception of <code class="docutils literal notranslate"><span class="pre">permutation</span></code>, the parameters and storage |
| of VariableShapeTensor relate to the <em>physical</em> storage of the tensor.</p> |
| <dl class="simple"> |
| <dt>For example, consider a tensor with::</dt><dd><p>shape = [10, 20, 30] |
| dim_names = [x, y, z] |
| permutations = [2, 0, 1]</p> |
| </dd> |
| </dl> |
| <p>This means the logical tensor has names [z, x, y] and shape [30, 10, 20].</p> |
| </div> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Values inside each <strong>data</strong> tensor element are stored in row-major/C-contiguous |
| order according to the corresponding <strong>shape</strong>.</p> |
| </div> |
| </section> |
| </section> |
| <section id="community-extension-types"> |
| <h2>Community Extension Types<a class="headerlink" href="#community-extension-types" title="Permalink to this heading">#</a></h2> |
| <p>In addition to the canonical extension types listed above, there exist Arrow |
| extension types that have been established as standards within specific domain |
| areas. These have not been officially designated as canonical through a |
| discussion and vote on the Arrow development mailing list but are well known |
| within subcommunities of Arrow developers.</p> |
| <section id="geoarrow"> |
| <h3>GeoArrow<a class="headerlink" href="#geoarrow" title="Permalink to this heading">#</a></h3> |
| <p><a class="reference external" href="https://github.com/geoarrow/geoarrow">GeoArrow</a> defines a collection of |
| Arrow extension types for representing vector geometries. It is well known |
| within the Arrow geospatial subcommunity. The GeoArrow specification is not yet |
| finalized.</p> |
| </section> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="Columnar.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Arrow Columnar Format</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="Other.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">Other Data Structures</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#introduction">Introduction</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#standardization">Standardization</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#making-modifications">Making Modifications</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#official-list">Official List</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#fixed-shape-tensor">Fixed shape tensor</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#variable-shape-tensor">Variable shape tensor</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#community-extension-types">Community Extension Types</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#geoarrow">GeoArrow</a></li> |
| </ul> |
| </li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/format/CanonicalExtensions.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script> |
| <script src="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2024 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
