| |
| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" /> |
| |
| <title>The Arrow C data interface — Apache Arrow v17.0.0.dev52</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; |
| </script> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| |
| |
| <link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> |
| |
| <link rel="stylesheet" type="text/css" href="../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> |
| <link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> |
| <script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script> |
| <script src="../_static/doctools.js"></script> |
| <script src="../_static/sphinx_highlight.js"></script> |
| <script src="../_static/clipboard.min.js"></script> |
| <script src="../_static/copybutton.js"></script> |
| <script src="../_static/design-tabs.js"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = 'format/CDataInterface';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.15.2'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/format/CDataInterface.html" /> |
| <link rel="icon" href="../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="The Arrow PyCapsule Interface" href="CDataInterface/PyCapsuleInterface.html" /> |
| <link rel="prev" title="Other Data Structures" href="Other.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i> |
| Back to top |
| </button> |
| |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__primary" |
| id="__primary"/> |
| <label class="overlay overlay-primary" for="__primary"></label> |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__secondary" |
| id="__secondary"/> |
| <label class="overlay overlay-secondary" for="__secondary"></label> |
| |
| <div class="search-button__wrapper"> |
| <div class="search-button__overlay"></div> |
| <div class="search-button__search-container"> |
| <form class="bd-search d-flex align-items-center" |
| action="../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| id="search-input" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form></div> |
| </div> |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar"> |
| <div class="bd-header__inner bd-page-width"> |
| <label class="sidebar-toggle primary-toggle" for="__primary"> |
| <span class="fa-solid fa-bars"></span> |
| </label> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| <a class="navbar-brand logo" href="../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev52 - Home"/> |
| <script>document.write(`<img src="../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev52 - Home"/>`);</script> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| |
| <label class="sidebar-toggle secondary-toggle" for="__secondary" tabindex="0"> |
| <span class="fa-solid fa-outdent"></span> |
| </label> |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| <div class="bd-sidebar-primary bd-sidebar"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item current active"> |
| <a class="nav-link nav-internal" href="index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links-2" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| <div class="sidebar-primary-items__start sidebar-primary__section"> |
| <div class="sidebar-primary-item"> |
| <nav class="bd-docs-nav bd-links" |
| aria-label="Section Navigation"> |
| <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p> |
| <div class="bd-toc-item navbar-nav"><ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"><a class="reference internal" href="Versioning.html">Format Versioning and Stability</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Columnar.html">Arrow Columnar Format</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="CanonicalExtensions.html">Canonical Extension Types</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Other.html">Other Data Structures</a></li> |
| <li class="toctree-l1 current active has-children"><a class="current reference internal" href="#">The Arrow C data interface</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-1"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="CDataInterface/PyCapsuleInterface.html">The Arrow PyCapsule Interface</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="CStreamInterface.html">The Arrow C stream interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="CDeviceDataInterface.html">The Arrow C Device data interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Flight.html">Arrow Flight RPC</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="FlightSql.html">Arrow Flight SQL</a></li> |
| <li class="toctree-l1 has-children"><a class="reference internal" href="ADBC.html">ADBC: Arrow Database Connectivity</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/><label class="toctree-toggle" for="toctree-checkbox-2"><i class="fa-solid fa-chevron-down"></i></label><ul> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/C.html">ADBC C API Specification</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/Go.html">ADBC Go API Specification</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="ADBC/Java.html">ADBC Java API Specification</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="Changing.html">Changing the Apache Arrow Format Specification</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Integration.html">Integration Testing</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="Glossary.html">Glossary</a></li> |
| </ul> |
| </div> |
| </nav></div> |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| </div> |
| |
| <div id="rtd-footer-container"></div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| |
| |
| <nav aria-label="Breadcrumb"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="index.html" class="nav-link">Specifications</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page">The Arrow C...</li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <section id="the-arrow-c-data-interface"> |
| <span id="c-data-interface"></span><h1>The Arrow C data interface<a class="headerlink" href="#the-arrow-c-data-interface" title="Permalink to this heading">#</a></h1> |
| <section id="rationale"> |
| <h2>Rationale<a class="headerlink" href="#rationale" title="Permalink to this heading">#</a></h2> |
| <p>Apache Arrow is designed to be a universal in-memory format for the representation |
| of tabular (“columnar”) data. However, some projects may face a difficult |
| choice between either depending on a fast-evolving project such as the |
| Arrow C++ library, or having to reimplement adapters for data interchange, |
| which may require significant, redundant development effort.</p> |
| <p>The Arrow C data interface defines a very small, stable set of C definitions |
| that can be easily <em>copied</em> in any project’s source code and used for columnar |
| data interchange in the Arrow format. For non-C/C++ languages and runtimes, |
| it should be almost as easy to translate the C definitions into the |
| corresponding C FFI declarations.</p> |
| <p>Applications and libraries can therefore work with Arrow memory without |
| necessarily using Arrow libraries or reinventing the wheel. Developers can |
| choose between tight integration |
| with the Arrow <em>software project</em> (benefiting from the growing array of |
| facilities exposed by e.g. the C++ or Java implementations of Apache Arrow, |
| but with the cost of a dependency) or minimal integration with the Arrow |
| <em>format</em> only.</p> |
| <section id="goals"> |
| <h3>Goals<a class="headerlink" href="#goals" title="Permalink to this heading">#</a></h3> |
| <ul class="simple"> |
| <li><p>Expose an ABI-stable interface.</p></li> |
| <li><p>Make it easy for third-party projects to implement support for (including partial |
| support where sufficient), with little initial investment.</p></li> |
| <li><p>Allow zero-copy sharing of Arrow data between independent runtimes |
| and components running in the same process.</p></li> |
| <li><p>Match the Arrow array concepts closely to avoid the development of |
| yet another marshalling layer.</p></li> |
| <li><p>Avoid the need for one-to-one adaptation layers such as the limited |
| JPype-based bridge between Java and Python.</p></li> |
| <li><p>Enable integration without an explicit dependency (either at compile-time |
| or runtime) on the Arrow software project.</p></li> |
| </ul> |
| <p>Ideally, the Arrow C data interface can become a low-level <em>lingua franca</em> |
| for sharing columnar data at runtime and establish Arrow as the universal |
| building block in the columnar processing ecosystem.</p> |
| </section> |
| <section id="non-goals"> |
| <h3>Non-goals<a class="headerlink" href="#non-goals" title="Permalink to this heading">#</a></h3> |
| <ul class="simple"> |
| <li><p>Expose a C API mimicking operations available in higher-level runtimes |
| (such as C++, Java…).</p></li> |
| <li><p>Data sharing between distinct processes or storage persistence.</p></li> |
| </ul> |
| </section> |
| <section id="comparison-with-the-arrow-ipc-format"> |
| <h3>Comparison with the Arrow IPC format<a class="headerlink" href="#comparison-with-the-arrow-ipc-format" title="Permalink to this heading">#</a></h3> |
| <p>Pros of the C data interface vs. the IPC format:</p> |
| <ul class="simple"> |
| <li><p>No dependency on Flatbuffers.</p></li> |
| <li><p>No buffer reassembly (data is already exposed in logical Arrow format).</p></li> |
| <li><p>Zero-copy by design.</p></li> |
| <li><p>Easy to reimplement from scratch.</p></li> |
| <li><p>Minimal C definition that can be easily copied into other codebases.</p></li> |
| <li><p>Resource lifetime management through a custom release callback.</p></li> |
| </ul> |
| <p>Pros of the IPC format vs. the data interface:</p> |
| <ul class="simple"> |
| <li><p>Works across processes and machines.</p></li> |
| <li><p>Allows data storage and persistence.</p></li> |
| <li><p>Being a streamable format, the IPC format has room for composing more features |
| (such as integrity checks, compression…).</p></li> |
| <li><p>Does not require explicit C data access.</p></li> |
| </ul> |
| </section> |
| </section> |
| <section id="data-type-description-format-strings"> |
| <h2>Data type description – format strings<a class="headerlink" href="#data-type-description-format-strings" title="Permalink to this heading">#</a></h2> |
| <p>A data type is described using a format string. The format string only |
| encodes information about the top-level type; for nested type, child types |
| are described separately. Also, metadata is encoded in a separate string.</p> |
| <p>The format strings are designed to be easily parsable, even from a language |
| such as C. The most common primitive formats have one-character format |
| strings:</p> |
| <table class="table"> |
| <thead> |
| <tr class="row-odd"><th class="head"><p>Format string</p></th> |
| <th class="head"><p>Arrow data type</p></th> |
| <th class="head"><p>Notes</p></th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">n</span></code></p></td> |
| <td><p>null</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">b</span></code></p></td> |
| <td><p>boolean</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">c</span></code></p></td> |
| <td><p>int8</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">C</span></code></p></td> |
| <td><p>uint8</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">s</span></code></p></td> |
| <td><p>int16</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">S</span></code></p></td> |
| <td><p>uint16</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">i</span></code></p></td> |
| <td><p>int32</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">I</span></code></p></td> |
| <td><p>uint32</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">l</span></code></p></td> |
| <td><p>int64</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">L</span></code></p></td> |
| <td><p>uint64</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">e</span></code></p></td> |
| <td><p>float16</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">f</span></code></p></td> |
| <td><p>float32</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">g</span></code></p></td> |
| <td><p>float64</p></td> |
| <td></td> |
| </tr> |
| </tbody> |
| </table> |
| <table class="table"> |
| <thead> |
| <tr class="row-odd"><th class="head"><p>Format string</p></th> |
| <th class="head"><p>Arrow data type</p></th> |
| <th class="head"><p>Notes</p></th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">z</span></code></p></td> |
| <td><p>binary</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">Z</span></code></p></td> |
| <td><p>large binary</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">vz</span></code></p></td> |
| <td><p>binary view</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">u</span></code></p></td> |
| <td><p>utf-8 string</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">U</span></code></p></td> |
| <td><p>large utf-8 string</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">vu</span></code></p></td> |
| <td><p>utf-8 view</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">d:19,10</span></code></p></td> |
| <td><p>decimal128 [precision 19, scale 10]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">d:19,10,NNN</span></code></p></td> |
| <td><p>decimal bitwidth = NNN [precision 19, scale 10]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">w:42</span></code></p></td> |
| <td><p>fixed-width binary [42 bytes]</p></td> |
| <td></td> |
| </tr> |
| </tbody> |
| </table> |
| <p>Temporal types have multi-character format strings starting with <code class="docutils literal notranslate"><span class="pre">t</span></code>:</p> |
| <table class="table"> |
| <thead> |
| <tr class="row-odd"><th class="head"><p>Format string</p></th> |
| <th class="head"><p>Arrow data type</p></th> |
| <th class="head"><p>Notes</p></th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tdD</span></code></p></td> |
| <td><p>date32 [days]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">tdm</span></code></p></td> |
| <td><p>date64 [milliseconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tts</span></code></p></td> |
| <td><p>time32 [seconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">ttm</span></code></p></td> |
| <td><p>time32 [milliseconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">ttu</span></code></p></td> |
| <td><p>time64 [microseconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">ttn</span></code></p></td> |
| <td><p>time64 [nanoseconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tss:...</span></code></p></td> |
| <td><p>timestamp [seconds] with timezone “…”</p></td> |
| <td><p>(1)</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">tsm:...</span></code></p></td> |
| <td><p>timestamp [milliseconds] with timezone “…”</p></td> |
| <td><p>(1)</p></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tsu:...</span></code></p></td> |
| <td><p>timestamp [microseconds] with timezone “…”</p></td> |
| <td><p>(1)</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">tsn:...</span></code></p></td> |
| <td><p>timestamp [nanoseconds] with timezone “…”</p></td> |
| <td><p>(1)</p></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tDs</span></code></p></td> |
| <td><p>duration [seconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">tDm</span></code></p></td> |
| <td><p>duration [milliseconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tDu</span></code></p></td> |
| <td><p>duration [microseconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">tDn</span></code></p></td> |
| <td><p>duration [nanoseconds]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tiM</span></code></p></td> |
| <td><p>interval [months]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">tiD</span></code></p></td> |
| <td><p>interval [days, time]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">tin</span></code></p></td> |
| <td><p>interval [month, day, nanoseconds]</p></td> |
| <td></td> |
| </tr> |
| </tbody> |
| </table> |
| <p>Dictionary-encoded types do not have a specific format string. Instead, the |
| format string of the base array represents the dictionary index type, and the |
| value type can be read from the dependent dictionary array (see below |
| “Dictionary-encoded arrays”).</p> |
| <p>Nested types have multiple-character format strings starting with <code class="docutils literal notranslate"><span class="pre">+</span></code>. The |
| names and types of child fields are read from the child arrays.</p> |
| <table class="table"> |
| <thead> |
| <tr class="row-odd"><th class="head"><p>Format string</p></th> |
| <th class="head"><p>Arrow data type</p></th> |
| <th class="head"><p>Notes</p></th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">+l</span></code></p></td> |
| <td><p>list</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">+L</span></code></p></td> |
| <td><p>large list</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">+vl</span></code></p></td> |
| <td><p>list-view</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">+vL</span></code></p></td> |
| <td><p>large list-view</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">+w:123</span></code></p></td> |
| <td><p>fixed-sized list [123 items]</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">+s</span></code></p></td> |
| <td><p>struct</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">+m</span></code></p></td> |
| <td><p>map</p></td> |
| <td><p>(2)</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">+ud:I,J,...</span></code></p></td> |
| <td><p>dense union with type ids I,J…</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">+us:I,J,...</span></code></p></td> |
| <td><p>sparse union with type ids I,J…</p></td> |
| <td></td> |
| </tr> |
| <tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">+r</span></code></p></td> |
| <td><p>run-end encoded</p></td> |
| <td><p>(3)</p></td> |
| </tr> |
| </tbody> |
| </table> |
| <p>Notes:</p> |
| <ol class="arabic simple"> |
| <li><p>The timezone string is appended as-is after the colon character <code class="docutils literal notranslate"><span class="pre">:</span></code>, without |
| any quotes. If the timezone is empty, the colon <code class="docutils literal notranslate"><span class="pre">:</span></code> must still be included.</p></li> |
| <li><p>As specified in the Arrow columnar format, the map type has a single child type |
| named <code class="docutils literal notranslate"><span class="pre">entries</span></code>, itself a 2-child struct type of <code class="docutils literal notranslate"><span class="pre">(key,</span> <span class="pre">value)</span></code>.</p></li> |
| <li><p>As specified in the Arrow columnar format, the run-end encoded type has two |
| children where the first is the (integral) <code class="docutils literal notranslate"><span class="pre">run_ends</span></code> and the second is the |
| <code class="docutils literal notranslate"><span class="pre">values</span></code>.</p></li> |
| </ol> |
| <section id="examples"> |
| <h3>Examples<a class="headerlink" href="#examples" title="Permalink to this heading">#</a></h3> |
| <ul class="simple"> |
| <li><p>A dictionary-encoded <code class="docutils literal notranslate"><span class="pre">decimal128(precision</span> <span class="pre">=</span> <span class="pre">12,</span> <span class="pre">scale</span> <span class="pre">=</span> <span class="pre">5)</span></code> array |
| with <code class="docutils literal notranslate"><span class="pre">int16</span></code> indices has format string <code class="docutils literal notranslate"><span class="pre">s</span></code>, and its dependent dictionary |
| array has format string <code class="docutils literal notranslate"><span class="pre">d:12,5</span></code>.</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">list<uint64></span></code> array has format string <code class="docutils literal notranslate"><span class="pre">+l</span></code>, and its single child |
| has format string <code class="docutils literal notranslate"><span class="pre">L</span></code>.</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">large_list_view<uint64></span></code> array has format string <code class="docutils literal notranslate"><span class="pre">+vL</span></code>, and its single |
| child has format string <code class="docutils literal notranslate"><span class="pre">L</span></code>.</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">struct<ints:</span> <span class="pre">int32,</span> <span class="pre">floats:</span> <span class="pre">float32></span></code> has format string <code class="docutils literal notranslate"><span class="pre">+s</span></code>; its two |
| children have names <code class="docutils literal notranslate"><span class="pre">ints</span></code> and <code class="docutils literal notranslate"><span class="pre">floats</span></code>, and format strings <code class="docutils literal notranslate"><span class="pre">i</span></code> and |
| <code class="docutils literal notranslate"><span class="pre">f</span></code> respectively.</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">map<string,</span> <span class="pre">float64></span></code> array has format string <code class="docutils literal notranslate"><span class="pre">+m</span></code>; its single child |
| has name <code class="docutils literal notranslate"><span class="pre">entries</span></code> and format string <code class="docutils literal notranslate"><span class="pre">+s</span></code>; its two grandchildren have names |
| <code class="docutils literal notranslate"><span class="pre">key</span></code> and <code class="docutils literal notranslate"><span class="pre">value</span></code>, and format strings <code class="docutils literal notranslate"><span class="pre">u</span></code> and <code class="docutils literal notranslate"><span class="pre">g</span></code> respectively.</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">sparse_union<ints:</span> <span class="pre">int32,</span> <span class="pre">floats:</span> <span class="pre">float32></span></code> with type ids <code class="docutils literal notranslate"><span class="pre">4,</span> <span class="pre">5</span></code> |
| has format string <code class="docutils literal notranslate"><span class="pre">+us:4,5</span></code>; its two children have names <code class="docutils literal notranslate"><span class="pre">ints</span></code> and |
| <code class="docutils literal notranslate"><span class="pre">floats</span></code>, and format strings <code class="docutils literal notranslate"><span class="pre">i</span></code> and <code class="docutils literal notranslate"><span class="pre">f</span></code> respectively.</p></li> |
| <li><p>A <code class="docutils literal notranslate"><span class="pre">run_end_encoded<int32,</span> <span class="pre">float32></span></code> has format string <code class="docutils literal notranslate"><span class="pre">+r</span></code>; its two |
| children have names <code class="docutils literal notranslate"><span class="pre">run_ends</span></code> and <code class="docutils literal notranslate"><span class="pre">values</span></code>, and format strings |
| <code class="docutils literal notranslate"><span class="pre">i</span></code> and <code class="docutils literal notranslate"><span class="pre">f</span></code> respectively.</p></li> |
| </ul> |
| </section> |
| </section> |
| <section id="structure-definitions"> |
| <span id="c-data-interface-struct-defs"></span><h2>Structure definitions<a class="headerlink" href="#structure-definitions" title="Permalink to this heading">#</a></h2> |
| <p>The following free-standing definitions are enough to support the Arrow |
| C data interface in your project. Like the rest of the Arrow project, they |
| are available under the Apache License 2.0.</p> |
| <div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#ifndef ARROW_C_DATA_INTERFACE</span> |
| <span class="cp">#define ARROW_C_DATA_INTERFACE</span> |
| |
| <span class="cp">#define ARROW_FLAG_DICTIONARY_ORDERED 1</span> |
| <span class="cp">#define ARROW_FLAG_NULLABLE 2</span> |
| <span class="cp">#define ARROW_FLAG_MAP_KEYS_SORTED 4</span> |
| |
| <span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Array type description</span> |
| <span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">format</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">name</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">char</span><span class="o">*</span><span class="w"> </span><span class="n">metadata</span><span class="p">;</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">flags</span><span class="p">;</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">n_children</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">**</span><span class="w"> </span><span class="n">children</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="w"> </span><span class="n">dictionary</span><span class="p">;</span> |
| |
| <span class="w"> </span><span class="c1">// Release callback</span> |
| <span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="p">(</span><span class="o">*</span><span class="n">release</span><span class="p">)(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="p">);</span> |
| <span class="w"> </span><span class="c1">// Opaque producer-specific data</span> |
| <span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">private_data</span><span class="p">;</span> |
| <span class="p">};</span> |
| |
| <span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Array data description</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">length</span><span class="p">;</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">null_count</span><span class="p">;</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">offset</span><span class="p">;</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">n_buffers</span><span class="p">;</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">n_children</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">void</span><span class="o">**</span><span class="w"> </span><span class="n">buffers</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">**</span><span class="w"> </span><span class="n">children</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">dictionary</span><span class="p">;</span> |
| |
| <span class="w"> </span><span class="c1">// Release callback</span> |
| <span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="p">(</span><span class="o">*</span><span class="n">release</span><span class="p">)(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="p">);</span> |
| <span class="w"> </span><span class="c1">// Opaque producer-specific data</span> |
| <span class="w"> </span><span class="kt">void</span><span class="o">*</span><span class="w"> </span><span class="n">private_data</span><span class="p">;</span> |
| <span class="p">};</span> |
| |
| <span class="cp">#endif </span><span class="c1">// ARROW_C_DATA_INTERFACE</span> |
| </pre></div> |
| </div> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>The canonical guard <code class="docutils literal notranslate"><span class="pre">ARROW_C_DATA_INTERFACE</span></code> is meant to avoid |
| duplicate definitions if two projects copy the C data interface |
| definitions in their own headers, and a third-party project |
| includes from these two projects. It is therefore important that |
| this guard is kept exactly as-is when these definitions are copied.</p> |
| </div> |
| <section id="the-arrowschema-structure"> |
| <h3>The ArrowSchema structure<a class="headerlink" href="#the-arrowschema-structure" title="Permalink to this heading">#</a></h3> |
| <p>The <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> structure describes the type and metadata of an exported |
| array or record batch. It has the following fields:</p> |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.format"> |
| <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">format</span></span></span><a class="headerlink" href="#c.ArrowSchema.format" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. A null-terminated, UTF8-encoded string describing |
| the data type. If the data type is nested, child types are not |
| encoded here but in the <a class="reference internal" href="#c.ArrowSchema.children" title="ArrowSchema.children"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.children</span></code></a> structures.</p> |
| <p>Consumers MAY decide not to support all data types, but they |
| should document this limitation.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.name"> |
| <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><a class="headerlink" href="#c.ArrowSchema.name" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. A null-terminated, UTF8-encoded string of the field |
| or array name. This is mainly used to reconstruct child fields |
| of nested types.</p> |
| <p>Producers MAY decide not to provide this information, and consumers |
| MAY decide to ignore it. If omitted, MAY be NULL or an empty string.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.metadata"> |
| <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">metadata</span></span></span><a class="headerlink" href="#c.ArrowSchema.metadata" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. A binary string describing the type’s metadata. |
| If the data type is nested, child types are not encoded here but |
| in the <a class="reference internal" href="#c.ArrowSchema.children" title="ArrowSchema.children"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.children</span></code></a> structures.</p> |
| <p>This string is not null-terminated but follows a specific format:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">int32</span><span class="p">:</span> <span class="n">number</span> <span class="n">of</span> <span class="n">key</span><span class="o">/</span><span class="n">value</span> <span class="n">pairs</span> <span class="p">(</span><span class="n">noted</span> <span class="n">N</span> <span class="n">below</span><span class="p">)</span> |
| <span class="n">int32</span><span class="p">:</span> <span class="n">byte</span> <span class="n">length</span> <span class="n">of</span> <span class="n">key</span> <span class="mi">0</span> |
| <span class="n">key</span> <span class="mi">0</span> <span class="p">(</span><span class="ow">not</span> <span class="n">null</span><span class="o">-</span><span class="n">terminated</span><span class="p">)</span> |
| <span class="n">int32</span><span class="p">:</span> <span class="n">byte</span> <span class="n">length</span> <span class="n">of</span> <span class="n">value</span> <span class="mi">0</span> |
| <span class="n">value</span> <span class="mi">0</span> <span class="p">(</span><span class="ow">not</span> <span class="n">null</span><span class="o">-</span><span class="n">terminated</span><span class="p">)</span> |
| <span class="o">...</span> |
| <span class="n">int32</span><span class="p">:</span> <span class="n">byte</span> <span class="n">length</span> <span class="n">of</span> <span class="n">key</span> <span class="n">N</span> <span class="o">-</span> <span class="mi">1</span> |
| <span class="n">key</span> <span class="n">N</span> <span class="o">-</span> <span class="mi">1</span> <span class="p">(</span><span class="ow">not</span> <span class="n">null</span><span class="o">-</span><span class="n">terminated</span><span class="p">)</span> |
| <span class="n">int32</span><span class="p">:</span> <span class="n">byte</span> <span class="n">length</span> <span class="n">of</span> <span class="n">value</span> <span class="n">N</span> <span class="o">-</span> <span class="mi">1</span> |
| <span class="n">value</span> <span class="n">N</span> <span class="o">-</span> <span class="mi">1</span> <span class="p">(</span><span class="ow">not</span> <span class="n">null</span><span class="o">-</span><span class="n">terminated</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Integers are stored in native endianness. For example, the metadata |
| <code class="docutils literal notranslate"><span class="pre">[('key1',</span> <span class="pre">'value1')]</span></code> is encoded on a little-endian machine as:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>\<span class="n">x01</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x04</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x00key1</span>\<span class="n">x06</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x00value1</span> |
| </pre></div> |
| </div> |
| <p>On a big-endian machine, the same example would be encoded as:</p> |
| <div class="highlight-default notranslate"><div class="highlight"><pre><span></span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x01</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x04key1</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x00</span>\<span class="n">x06value1</span> |
| </pre></div> |
| </div> |
| <p>If omitted, this field MUST be NULL (not an empty string).</p> |
| <p>Consumers MAY choose to ignore this information.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.flags"> |
| <span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">flags</span></span></span><a class="headerlink" href="#c.ArrowSchema.flags" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. A bitfield of flags enriching the type description. |
| Its value is computed by OR’ing together the flag values. |
| The following flags are available:</p> |
| <ul class="simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">ARROW_FLAG_NULLABLE</span></code>: whether this field is semantically nullable |
| (regardless of whether it actually has null values).</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">ARROW_FLAG_DICTIONARY_ORDERED</span></code>: for dictionary-encoded types, |
| whether the ordering of dictionary indices is semantically meaningful.</p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">ARROW_FLAG_MAP_KEYS_SORTED</span></code>: for map types, whether the keys within |
| each map value are sorted.</p></li> |
| </ul> |
| <p>If omitted, MUST be 0.</p> |
| <p>Consumers MAY choose to ignore some or all of the flags. Even then, |
| they SHOULD keep this value around so as to propagate its information |
| to their own consumers.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.n_children"> |
| <span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">n_children</span></span></span><a class="headerlink" href="#c.ArrowSchema.n_children" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. The number of children this type has.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.children"> |
| <span class="n"><span class="pre">ArrowSchema</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">children</span></span></span><a class="headerlink" href="#c.ArrowSchema.children" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. A C array of pointers to each child type of this type. |
| There must be <a class="reference internal" href="#c.ArrowSchema.n_children" title="ArrowSchema.n_children"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.n_children</span></code></a> pointers.</p> |
| <p>MAY be NULL only if <a class="reference internal" href="#c.ArrowSchema.n_children" title="ArrowSchema.n_children"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.n_children</span></code></a> is 0.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.dictionary"> |
| <span class="n"><span class="pre">ArrowSchema</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">dictionary</span></span></span><a class="headerlink" href="#c.ArrowSchema.dictionary" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. A pointer to the type of dictionary values.</p> |
| <p>MUST be present if the ArrowSchema represents a dictionary-encoded type. |
| MUST be NULL otherwise.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.release"> |
| <span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">release</span></span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">(</span></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#c.ArrowSchema.release" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. A pointer to a producer-provided release callback.</p> |
| <p>See below for memory management and release callback semantics.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowSchema.private_data"> |
| <span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowSchema</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">private_data</span></span></span><a class="headerlink" href="#c.ArrowSchema.private_data" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. An opaque pointer to producer-provided private data.</p> |
| <p>Consumers MUST not process this member. Lifetime of this member |
| is handled by the producer, and especially by the release callback.</p> |
| </dd></dl> |
| |
| </section> |
| <section id="the-arrowarray-structure"> |
| <h3>The ArrowArray structure<a class="headerlink" href="#the-arrowarray-structure" title="Permalink to this heading">#</a></h3> |
| <p>The <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> describes the data of an exported array or record batch. |
| For the <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure to be interpreted type, the array type |
| or record batch schema must already be known. This is either done by |
| convention – for example a producer API that always produces the same data |
| type – or by passing a <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> on the side.</p> |
| <p>It has the following fields:</p> |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.length"> |
| <span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">length</span></span></span><a class="headerlink" href="#c.ArrowArray.length" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. The logical length of the array (i.e. its number of items).</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.null_count"> |
| <span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">null_count</span></span></span><a class="headerlink" href="#c.ArrowArray.null_count" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. The number of null items in the array. MAY be -1 if not |
| yet computed.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.offset"> |
| <span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">offset</span></span></span><a class="headerlink" href="#c.ArrowArray.offset" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. The logical offset inside the array (i.e. the number of items |
| from the physical start of the buffers). MUST be 0 or positive.</p> |
| <p>Producers MAY specify that they will only produce 0-offset arrays to |
| ease implementation of consumer code. |
| Consumers MAY decide not to support non-0-offset arrays, but they |
| should document this limitation.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.n_buffers"> |
| <span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">n_buffers</span></span></span><a class="headerlink" href="#c.ArrowArray.n_buffers" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. The number of physical buffers backing this array. The |
| number of buffers is a function of the data type, as described in the |
| <a class="reference internal" href="Columnar.html#format-columnar"><span class="std std-ref">Columnar format specification</span></a>, except for the |
| the binary or utf-8 view type, which has one additional buffer compared |
| to the Columnar format specification (see |
| <a class="reference internal" href="#c-data-interface-binary-view-arrays"><span class="std std-ref">Binary view arrays</span></a>).</p> |
| <p>Buffers of children arrays are not included.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.buffers"> |
| <span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">buffers</span></span></span><a class="headerlink" href="#c.ArrowArray.buffers" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. A C array of pointers to the start of each physical buffer |
| backing this array. Each <cite>void*</cite> pointer is the physical start of |
| a contiguous buffer. There must be <a class="reference internal" href="#c.ArrowArray.n_buffers" title="ArrowArray.n_buffers"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowArray.n_buffers</span></code></a> pointers.</p> |
| <p>The producer MUST ensure that each contiguous buffer is large enough to |
| represent <cite>length + offset</cite> values encoded according to the |
| <a class="reference internal" href="Columnar.html#format-columnar"><span class="std std-ref">Columnar format specification</span></a>.</p> |
| <p>It is recommended, but not required, that the memory addresses of the |
| buffers be aligned at least according to the type of primitive data that |
| they contain. Consumers MAY decide not to support unaligned memory.</p> |
| <p>The buffer pointers MAY be null only in two situations:</p> |
| <ol class="arabic simple"> |
| <li><p>for the null bitmap buffer, if <a class="reference internal" href="#c.ArrowArray.null_count" title="ArrowArray.null_count"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowArray.null_count</span></code></a> is 0;</p></li> |
| <li><p>for any buffer, if the size in bytes of the corresponding buffer would be 0.</p></li> |
| </ol> |
| <p>Buffers of children arrays are not included.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.n_children"> |
| <span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">n_children</span></span></span><a class="headerlink" href="#c.ArrowArray.n_children" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. The number of children this array has. The number of children |
| is a function of the data type, as described in the |
| <a class="reference internal" href="Columnar.html#format-columnar"><span class="std std-ref">Columnar format specification</span></a>.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.children"> |
| <span class="n"><span class="pre">ArrowArray</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">children</span></span></span><a class="headerlink" href="#c.ArrowArray.children" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. A C array of pointers to each child array of this array. |
| There must be <a class="reference internal" href="#c.ArrowArray.n_children" title="ArrowArray.n_children"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowArray.n_children</span></code></a> pointers.</p> |
| <p>MAY be NULL only if <a class="reference internal" href="#c.ArrowArray.n_children" title="ArrowArray.n_children"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowArray.n_children</span></code></a> is 0.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.dictionary"> |
| <span class="n"><span class="pre">ArrowArray</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">dictionary</span></span></span><a class="headerlink" href="#c.ArrowArray.dictionary" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. A pointer to the underlying array of dictionary values.</p> |
| <p>MUST be present if the ArrowArray represents a dictionary-encoded array. |
| MUST be NULL otherwise.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.release"> |
| <span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">release</span></span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">(</span></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#c.ArrowArray.release" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Mandatory. A pointer to a producer-provided release callback.</p> |
| <p>See below for memory management and release callback semantics.</p> |
| </dd></dl> |
| |
| <dl class="c member"> |
| <dt class="sig sig-object c" id="c.ArrowArray.private_data"> |
| <span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-prename descclassname"><span class="n"><span class="pre">ArrowArray</span></span><span class="p"><span class="pre">.</span></span></span><span class="sig-name descname"><span class="n"><span class="pre">private_data</span></span></span><a class="headerlink" href="#c.ArrowArray.private_data" title="Permalink to this definition">#</a><br /></dt> |
| <dd><p>Optional. An opaque pointer to producer-provided private data.</p> |
| <p>Consumers MUST not process this member. Lifetime of this member |
| is handled by the producer, and especially by the release callback.</p> |
| </dd></dl> |
| |
| </section> |
| <section id="dictionary-encoded-arrays"> |
| <h3>Dictionary-encoded arrays<a class="headerlink" href="#dictionary-encoded-arrays" title="Permalink to this heading">#</a></h3> |
| <p>For dictionary-encoded arrays, the <a class="reference internal" href="#c.ArrowSchema.format" title="ArrowSchema.format"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.format</span></code></a> string |
| encodes the <em>index</em> type. The dictionary <em>value</em> type can be read |
| from the <a class="reference internal" href="#c.ArrowSchema.dictionary" title="ArrowSchema.dictionary"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.dictionary</span></code></a> structure.</p> |
| <p>The same holds for <code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure: while the parent |
| structure points to the index data, the <a class="reference internal" href="#c.ArrowArray.dictionary" title="ArrowArray.dictionary"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowArray.dictionary</span></code></a> |
| points to the dictionary values array.</p> |
| </section> |
| <section id="extension-arrays"> |
| <h3>Extension arrays<a class="headerlink" href="#extension-arrays" title="Permalink to this heading">#</a></h3> |
| <p>For extension arrays, the <a class="reference internal" href="#c.ArrowSchema.format" title="ArrowSchema.format"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.format</span></code></a> string encodes the |
| <em>storage</em> type. Information about the extension type is encoded in the |
| <a class="reference internal" href="#c.ArrowSchema.metadata" title="ArrowSchema.metadata"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.metadata</span></code></a> string, similarly to the |
| <a class="reference internal" href="Columnar.html#format-metadata-extension-types"><span class="std std-ref">IPC format</span></a>. Specifically, the |
| metadata key <code class="docutils literal notranslate"><span class="pre">ARROW:extension:name</span></code> encodes the extension type name, |
| and the metadata key <code class="docutils literal notranslate"><span class="pre">ARROW:extension:metadata</span></code> encodes the |
| implementation-specific serialization of the extension type (for |
| parameterized extension types).</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure exported from an extension array simply points |
| to the storage data of the extension array.</p> |
| </section> |
| <section id="binary-view-arrays"> |
| <span id="c-data-interface-binary-view-arrays"></span><h3>Binary view arrays<a class="headerlink" href="#binary-view-arrays" title="Permalink to this heading">#</a></h3> |
| <p>For binary or utf-8 view arrays, an extra buffer is appended which stores |
| the lengths of each variadic data buffer as <code class="docutils literal notranslate"><span class="pre">int64_t</span></code>. This buffer is |
| necessary since these buffer lengths are not trivially extractable from |
| other data in an array of binary or utf-8 view type.</p> |
| </section> |
| </section> |
| <section id="semantics"> |
| <span id="c-data-interface-semantics"></span><h2>Semantics<a class="headerlink" href="#semantics" title="Permalink to this heading">#</a></h2> |
| <section id="memory-management"> |
| <h3>Memory management<a class="headerlink" href="#memory-management" title="Permalink to this heading">#</a></h3> |
| <p>The <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> and <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structures follow the same conventions |
| for memory management. The term <em>“base structure”</em> below refers to the |
| <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> or <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> that is passed between producer and consumer |
| – not any child structure thereof.</p> |
| <section id="member-allocation"> |
| <h4>Member allocation<a class="headerlink" href="#member-allocation" title="Permalink to this heading">#</a></h4> |
| <p>It is intended for the base structure to be stack- or heap-allocated by the |
| consumer. In this case, the producer API should take a pointer to the |
| consumer-allocated structure.</p> |
| <p>However, any data pointed to by the struct MUST be allocated and maintained |
| by the producer. This includes the format and metadata strings, the arrays |
| of buffer and children pointers, etc.</p> |
| <p>Therefore, the consumer MUST not try to interfere with the producer’s |
| handling of these members’ lifetime. The only way the consumer influences |
| data lifetime is by calling the base structure’s <code class="docutils literal notranslate"><span class="pre">release</span></code> callback.</p> |
| </section> |
| <section id="released-structure"> |
| <span id="c-data-interface-released"></span><h4>Released structure<a class="headerlink" href="#released-structure" title="Permalink to this heading">#</a></h4> |
| <p>A released structure is indicated by setting its <code class="docutils literal notranslate"><span class="pre">release</span></code> callback to NULL. |
| Before reading and interpreting a structure’s data, consumers SHOULD check |
| for a NULL release callback and treat it accordingly (probably by erroring |
| out).</p> |
| </section> |
| <section id="release-callback-semantics-for-consumers"> |
| <h4>Release callback semantics – for consumers<a class="headerlink" href="#release-callback-semantics-for-consumers" title="Permalink to this heading">#</a></h4> |
| <p>Consumers MUST call a base structure’s release callback when they won’t be using |
| it anymore, but they MUST not call any of its children’s release callbacks |
| (including the optional dictionary). The producer is responsible for releasing |
| the children.</p> |
| <p>In any case, a consumer MUST not try to access the base structure anymore |
| after calling its release callback – including any associated data such |
| as its children.</p> |
| </section> |
| <section id="release-callback-semantics-for-producers"> |
| <h4>Release callback semantics – for producers<a class="headerlink" href="#release-callback-semantics-for-producers" title="Permalink to this heading">#</a></h4> |
| <p>If producers need additional information for lifetime handling (for |
| example, a C++ producer may want to use <code class="docutils literal notranslate"><span class="pre">shared_ptr</span></code> for array and |
| buffer lifetime), they MUST use the <code class="docutils literal notranslate"><span class="pre">private_data</span></code> member to locate the |
| required bookkeeping information.</p> |
| <p>The release callback MUST not assume that the structure will be located |
| at the same memory location as when it was originally produced. The consumer |
| is free to move the structure around (see “Moving an array”).</p> |
| <p>The release callback MUST walk all children structures (including the optional |
| dictionary) and call their own release callbacks.</p> |
| <p>The release callback MUST free any data area directly owned by the structure |
| (such as the buffers and children members).</p> |
| <p>The release callback MUST mark the structure as released, by setting |
| its <code class="docutils literal notranslate"><span class="pre">release</span></code> member to NULL.</p> |
| <p>Below is a good starting point for implementing a release callback, where the |
| TODO area must be filled with producer-specific deallocation code:</p> |
| <div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">ReleaseExportedArray</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// This should not be called on already released array</span> |
| <span class="w"> </span><span class="n">assert</span><span class="p">(</span><span class="n">array</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">);</span> |
| |
| <span class="w"> </span><span class="c1">// Release children</span> |
| <span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int64_t</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">n_children</span><span class="p">;</span><span class="w"> </span><span class="o">++</span><span class="n">i</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">children</span><span class="p">[</span><span class="n">i</span><span class="p">];</span> |
| <span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">child</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">release</span><span class="p">(</span><span class="n">child</span><span class="p">);</span> |
| <span class="w"> </span><span class="n">assert</span><span class="p">(</span><span class="n">child</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="nb">NULL</span><span class="p">);</span> |
| <span class="w"> </span><span class="p">}</span> |
| <span class="w"> </span><span class="p">}</span> |
| |
| <span class="w"> </span><span class="c1">// Release dictionary</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">dict</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">dictionary</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">dict</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="nb">NULL</span><span class="w"> </span><span class="o">&&</span><span class="w"> </span><span class="n">dict</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="n">dict</span><span class="o">-></span><span class="n">release</span><span class="p">(</span><span class="n">dict</span><span class="p">);</span> |
| <span class="w"> </span><span class="n">assert</span><span class="p">(</span><span class="n">dict</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="nb">NULL</span><span class="p">);</span> |
| <span class="w"> </span><span class="p">}</span> |
| |
| <span class="w"> </span><span class="c1">// TODO here: release and/or deallocate all data directly owned by</span> |
| <span class="w"> </span><span class="c1">// the ArrowArray struct, such as the private_data.</span> |
| |
| <span class="w"> </span><span class="c1">// Mark array released</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">;</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="moving-an-array"> |
| <h4>Moving an array<a class="headerlink" href="#moving-an-array" title="Permalink to this heading">#</a></h4> |
| <p>The consumer can <em>move</em> the <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure by bitwise copying or |
| shallow member-wise copying. Then it MUST mark the source structure released |
| (see “released structure” above for how to do it) but <em>without</em> calling the |
| release callback. This ensures that only one live copy of the struct is |
| active at any given time and that lifetime is correctly communicated to |
| the producer.</p> |
| <p>As usual, the release callback will be called on the destination structure |
| when it is not needed anymore.</p> |
| <section id="moving-child-arrays"> |
| <h5>Moving child arrays<a class="headerlink" href="#moving-child-arrays" title="Permalink to this heading">#</a></h5> |
| <p>It is also possible to move one or several child arrays, but the parent |
| <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure MUST be released immediately afterwards, as it |
| won’t point to valid child arrays anymore.</p> |
| <p>The main use case for this is to keep alive only a subset of child arrays |
| (for example if you are only interested in certain columns of the data), |
| while releasing the others.</p> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>For moving to work correctly, the <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure has to be |
| trivially relocatable. Therefore, pointer members inside the <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> |
| structure (including <code class="docutils literal notranslate"><span class="pre">private_data</span></code>) MUST not point inside the structure |
| itself. Also, external pointers to the structure MUST not be separately |
| stored by the producer. Instead, the producer MUST use the <code class="docutils literal notranslate"><span class="pre">private_data</span></code> |
| member so as to remember any necessary bookkeeping information.</p> |
| </div> |
| </section> |
| </section> |
| </section> |
| <section id="record-batches"> |
| <h3>Record batches<a class="headerlink" href="#record-batches" title="Permalink to this heading">#</a></h3> |
| <p>A record batch can be trivially considered as an equivalent struct array. In |
| this case the metadata of the top-level <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> can be used for the |
| schema-level metadata of the record batch.</p> |
| </section> |
| <section id="mutability"> |
| <h3>Mutability<a class="headerlink" href="#mutability" title="Permalink to this heading">#</a></h3> |
| <p>Both the producer and the consumer SHOULD consider the exported data |
| (that is, the data reachable through the <code class="docutils literal notranslate"><span class="pre">buffers</span></code> member of <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code>) |
| to be immutable, as either party could otherwise see inconsistent data while |
| the other is mutating it.</p> |
| </section> |
| </section> |
| <section id="example-use-case"> |
| <h2>Example use case<a class="headerlink" href="#example-use-case" title="Permalink to this heading">#</a></h2> |
| <p>A C++ database engine wants to provide the option to deliver results in Arrow |
| format, but without imposing themselves a dependency on the Arrow software |
| libraries. With the Arrow C data interface, the engine can let the caller pass |
| a pointer to a <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure, and fill it with the next chunk of |
| results.</p> |
| <p>It can do so without including the Arrow C++ headers or linking with the |
| Arrow DLLs. Furthermore, the database engine’s C API can benefit other |
| runtimes and libraries that know about the Arrow C data interface, |
| through e.g. a C FFI layer.</p> |
| </section> |
| <section id="c-producer-examples"> |
| <h2>C producer examples<a class="headerlink" href="#c-producer-examples" title="Permalink to this heading">#</a></h2> |
| <section id="exporting-a-simple-int32-array"> |
| <h3>Exporting a simple <code class="docutils literal notranslate"><span class="pre">int32</span></code> array<a class="headerlink" href="#exporting-a-simple-int32-array" title="Permalink to this heading">#</a></h3> |
| <p id="c-data-interface-export-int32-schema">Export a non-nullable <code class="docutils literal notranslate"><span class="pre">int32</span></code> type with empty metadata. In this case, |
| all <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> members point to statically-allocated data, so the |
| release callback is trivial.</p> |
| <div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">release_int32_type</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Mark released</span> |
| <span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">;</span> |
| <span class="p">}</span> |
| |
| <span class="kt">void</span><span class="w"> </span><span class="nf">export_int32_type</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">schema</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Type description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">format</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"i"</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">""</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">metadata</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">flags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_int32_type</span> |
| <span class="w"> </span><span class="p">};</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>Export a C-malloc()ed array of the same type as a Arrow array, transferring |
| ownership to the consumer through the release callback:</p> |
| <div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">release_int32_array</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="n">assert</span><span class="p">(</span><span class="n">array</span><span class="o">-></span><span class="n">n_buffers</span><span class="w"> </span><span class="o">==</span><span class="w"> </span><span class="mi">2</span><span class="p">);</span> |
| <span class="w"> </span><span class="c1">// Free the buffers and the buffers array</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">((</span><span class="kt">void</span><span class="w"> </span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">1</span><span class="p">]);</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">(</span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="p">);</span> |
| <span class="w"> </span><span class="c1">// Mark released</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">;</span> |
| <span class="p">}</span> |
| |
| <span class="kt">void</span><span class="w"> </span><span class="nf">export_int32_array</span><span class="p">(</span><span class="k">const</span><span class="w"> </span><span class="kt">int32_t</span><span class="o">*</span><span class="w"> </span><span class="n">data</span><span class="p">,</span><span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">nitems</span><span class="p">,</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Initialize primitive fields</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">array</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Data description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">length</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">nitems</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">offset</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">null_count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_int32_array</span> |
| <span class="w"> </span><span class="p">};</span> |
| <span class="w"> </span><span class="c1">// Allocate list of buffers</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">const</span><span class="w"> </span><span class="kt">void</span><span class="o">**</span><span class="p">)</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="kt">void</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">n_buffers</span><span class="p">);</span> |
| <span class="w"> </span><span class="n">assert</span><span class="p">(</span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">);</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">;</span><span class="w"> </span><span class="c1">// no nulls, null bitmap can be omitted</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">data</span><span class="p">;</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </section> |
| <section id="exporting-a-struct-float32-utf8-array"> |
| <h3>Exporting a <code class="docutils literal notranslate"><span class="pre">struct<float32,</span> <span class="pre">utf8></span></code> array<a class="headerlink" href="#exporting-a-struct-float32-utf8-array" title="Permalink to this heading">#</a></h3> |
| <p>Export the array type as a <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> with C-malloc()ed children:</p> |
| <div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">release_malloced_type</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="p">;</span> |
| <span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">n_children</span><span class="p">;</span><span class="w"> </span><span class="o">++</span><span class="n">i</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="w"> </span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">children</span><span class="p">[</span><span class="n">i</span><span class="p">];</span> |
| <span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">child</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">release</span><span class="p">(</span><span class="n">child</span><span class="p">);</span> |
| <span class="w"> </span><span class="p">}</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">(</span><span class="n">child</span><span class="p">);</span> |
| <span class="w"> </span><span class="p">}</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">(</span><span class="n">schema</span><span class="o">-></span><span class="n">children</span><span class="p">);</span> |
| <span class="w"> </span><span class="c1">// Mark released</span> |
| <span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">;</span> |
| <span class="p">}</span> |
| |
| <span class="kt">void</span><span class="w"> </span><span class="nf">export_float32_utf8_type</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="w"> </span><span class="n">schema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="w"> </span><span class="n">child</span><span class="p">;</span> |
| |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="c1">// Initialize parent type</span> |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">schema</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Type description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">format</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"+s"</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">""</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">metadata</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">flags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_malloced_type</span> |
| <span class="w"> </span><span class="p">};</span> |
| <span class="w"> </span><span class="c1">// Allocate list of children types</span> |
| <span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">n_children</span><span class="p">);</span> |
| |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="c1">// Initialize child type #0</span> |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">children</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="p">));</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Type description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">format</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"f"</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"floats"</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">metadata</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">flags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ARROW_FLAG_NULLABLE</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_malloced_type</span> |
| <span class="w"> </span><span class="p">};</span> |
| |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="c1">// Initialize child type #1</span> |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">schema</span><span class="o">-></span><span class="n">children</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="p">));</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowSchema</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Type description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">format</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"u"</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">"strings"</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">metadata</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">flags</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">ARROW_FLAG_NULLABLE</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_malloced_type</span> |
| <span class="w"> </span><span class="p">};</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>Export C-malloc()ed arrays in Arrow-compatible layout as an Arrow struct array, |
| transferring ownership to the consumer:</p> |
| <div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">release_malloced_array</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="p">;</span> |
| <span class="w"> </span><span class="c1">// Free children</span> |
| <span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">n_children</span><span class="p">;</span><span class="w"> </span><span class="o">++</span><span class="n">i</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">children</span><span class="p">[</span><span class="n">i</span><span class="p">];</span> |
| <span class="w"> </span><span class="k">if</span><span class="w"> </span><span class="p">(</span><span class="n">child</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">!=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">release</span><span class="p">(</span><span class="n">child</span><span class="p">);</span> |
| <span class="w"> </span><span class="p">}</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">(</span><span class="n">child</span><span class="p">);</span> |
| <span class="w"> </span><span class="p">}</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">(</span><span class="n">array</span><span class="o">-></span><span class="n">children</span><span class="p">);</span> |
| <span class="w"> </span><span class="c1">// Free buffers</span> |
| <span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">n_buffers</span><span class="p">;</span><span class="w"> </span><span class="o">++</span><span class="n">i</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">((</span><span class="kt">void</span><span class="w"> </span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="n">i</span><span class="p">]);</span> |
| <span class="w"> </span><span class="p">}</span> |
| <span class="w"> </span><span class="n">free</span><span class="p">(</span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="p">);</span> |
| <span class="w"> </span><span class="c1">// Mark released</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">;</span> |
| <span class="p">}</span> |
| |
| <span class="kt">void</span><span class="w"> </span><span class="nf">export_float32_utf8_array</span><span class="p">(</span> |
| <span class="w"> </span><span class="kt">int64_t</span><span class="w"> </span><span class="n">nitems</span><span class="p">,</span> |
| <span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">uint8_t</span><span class="o">*</span><span class="w"> </span><span class="n">float32_nulls</span><span class="p">,</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">float</span><span class="o">*</span><span class="w"> </span><span class="n">float32_data</span><span class="p">,</span> |
| <span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">uint8_t</span><span class="o">*</span><span class="w"> </span><span class="n">utf8_nulls</span><span class="p">,</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">int32_t</span><span class="o">*</span><span class="w"> </span><span class="n">utf8_offsets</span><span class="p">,</span><span class="w"> </span><span class="k">const</span><span class="w"> </span><span class="kt">uint8_t</span><span class="o">*</span><span class="w"> </span><span class="n">utf8_data</span><span class="p">,</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="w"> </span><span class="n">child</span><span class="p">;</span> |
| |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="c1">// Initialize parent array</span> |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">array</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Data description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">length</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">nitems</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">offset</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">null_count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_malloced_array</span> |
| <span class="w"> </span><span class="p">};</span> |
| <span class="w"> </span><span class="c1">// Allocate list of parent buffers</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="kt">void</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">n_buffers</span><span class="p">);</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">;</span><span class="w"> </span><span class="c1">// no nulls, null bitmap can be omitted</span> |
| <span class="w"> </span><span class="c1">// Allocate list of children arrays</span> |
| <span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">n_children</span><span class="p">);</span> |
| |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="c1">// Initialize child array #0</span> |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">children</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="p">));</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Data description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">length</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">nitems</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">offset</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">null_count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">-1</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_malloced_array</span> |
| <span class="w"> </span><span class="p">};</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="kt">void</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">n_buffers</span><span class="p">);</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">float32_nulls</span><span class="p">;</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">float32_data</span><span class="p">;</span> |
| |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="c1">// Initialize child array #1</span> |
| <span class="w"> </span><span class="c1">//</span> |
| <span class="w"> </span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">array</span><span class="o">-></span><span class="n">children</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="p">));</span> |
| <span class="w"> </span><span class="o">*</span><span class="n">child</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">(</span><span class="k">struct</span><span class="w"> </span><span class="nc">ArrowArray</span><span class="p">)</span><span class="w"> </span><span class="p">{</span> |
| <span class="w"> </span><span class="c1">// Data description</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">length</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">nitems</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">offset</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">null_count</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">-1</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">n_children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">dictionary</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">children</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nb">NULL</span><span class="p">,</span> |
| <span class="w"> </span><span class="c1">// Bookkeeping</span> |
| <span class="w"> </span><span class="p">.</span><span class="n">release</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="o">&</span><span class="n">release_malloced_array</span> |
| <span class="w"> </span><span class="p">};</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">buffers</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">malloc</span><span class="p">(</span><span class="k">sizeof</span><span class="p">(</span><span class="kt">void</span><span class="o">*</span><span class="p">)</span><span class="w"> </span><span class="o">*</span><span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">n_buffers</span><span class="p">);</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">utf8_nulls</span><span class="p">;</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">utf8_offsets</span><span class="p">;</span> |
| <span class="w"> </span><span class="n">child</span><span class="o">-></span><span class="n">buffers</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">utf8_data</span><span class="p">;</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </section> |
| </section> |
| <section id="why-two-distinct-structures"> |
| <h2>Why two distinct structures?<a class="headerlink" href="#why-two-distinct-structures" title="Permalink to this heading">#</a></h2> |
| <p>In many cases, the same type or schema description applies to multiple, |
| possibly short, batches of data. To avoid paying the cost of exporting |
| and importing the type description for each batch, the <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> |
| can be passed once, separately, at the beginning of the conversation between |
| producer and consumer.</p> |
| <p>In other cases yet, the data type is fixed by the producer API, and may not |
| need to be communicated at all.</p> |
| <p>However, if a producer is focused on one-shot exchange of data, it can |
| communicate the <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> and <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structures in the same |
| API call.</p> |
| </section> |
| <section id="updating-this-specification"> |
| <h2>Updating this specification<a class="headerlink" href="#updating-this-specification" title="Permalink to this heading">#</a></h2> |
| <p>Once this specification is supported in an official Arrow release, the C |
| ABI is frozen. This means the <code class="docutils literal notranslate"><span class="pre">ArrowSchema</span></code> and <code class="docutils literal notranslate"><span class="pre">ArrowArray</span></code> structure |
| definitions should not change in any way – including adding new members.</p> |
| <p>Backwards-compatible changes are allowed, for example new |
| <a class="reference internal" href="#c.ArrowSchema.flags" title="ArrowSchema.flags"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.flags</span></code></a> values or expanded possibilities for |
| the <a class="reference internal" href="#c.ArrowSchema.format" title="ArrowSchema.format"><code class="xref c c-member docutils literal notranslate"><span class="pre">ArrowSchema.format</span></code></a> string.</p> |
| <p>Any incompatible changes should be part of a new specification, for example |
| “Arrow C data interface v2”.</p> |
| </section> |
| <section id="inspiration"> |
| <h2>Inspiration<a class="headerlink" href="#inspiration" title="Permalink to this heading">#</a></h2> |
| <p>The Arrow C data interface is inspired by the <a class="reference external" href="https://www.python.org/dev/peps/pep-3118/">Python buffer protocol</a>, |
| which has proven immensely successful in allowing various Python libraries |
| exchange numerical data with no knowledge of each other and near-zero |
| adaptation cost.</p> |
| </section> |
| <section id="language-specific-protocols"> |
| <h2>Language-specific protocols<a class="headerlink" href="#language-specific-protocols" title="Permalink to this heading">#</a></h2> |
| <p>Some languages may define additional protocols on top of the Arrow C data |
| interface.</p> |
| <div class="toctree-wrapper compound"> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="CDataInterface/PyCapsuleInterface.html">The Arrow PyCapsule Interface</a></li> |
| </ul> |
| </div> |
| </section> |
| </section> |
| |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer"> |
| |
| <div class="prev-next-area"> |
| <a class="left-prev" |
| href="Other.html" |
| title="previous page"> |
| <i class="fa-solid fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">Other Data Structures</p> |
| </div> |
| </a> |
| <a class="right-next" |
| href="CDataInterface/PyCapsuleInterface.html" |
| title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">The Arrow PyCapsule Interface</p> |
| </div> |
| <i class="fa-solid fa-angle-right"></i> |
| </a> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> |
| |
| |
| <div class="sidebar-secondary-item"> |
| <div |
| id="pst-page-navigation-heading-2" |
| class="page-toc tocsection onthispage"> |
| <i class="fa-solid fa-list"></i> On this page |
| </div> |
| <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2"> |
| <ul class="visible nav section-nav flex-column"> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rationale">Rationale</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#goals">Goals</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#non-goals">Non-goals</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#comparison-with-the-arrow-ipc-format">Comparison with the Arrow IPC format</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#data-type-description-format-strings">Data type description – format strings</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#examples">Examples</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#structure-definitions">Structure definitions</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#the-arrowschema-structure">The ArrowSchema structure</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#the-arrowarray-structure">The ArrowArray structure</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#dictionary-encoded-arrays">Dictionary-encoded arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#extension-arrays">Extension arrays</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#binary-view-arrays">Binary view arrays</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#semantics">Semantics</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#memory-management">Memory management</a><ul class="nav section-nav flex-column"> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#member-allocation">Member allocation</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#released-structure">Released structure</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#release-callback-semantics-for-consumers">Release callback semantics – for consumers</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#release-callback-semantics-for-producers">Release callback semantics – for producers</a></li> |
| <li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#moving-an-array">Moving an array</a><ul class="nav section-nav flex-column"> |
| <li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#moving-child-arrays">Moving child arrays</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#record-batches">Record batches</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#mutability">Mutability</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-use-case">Example use case</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#c-producer-examples">C producer examples</a><ul class="visible nav section-nav flex-column"> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#exporting-a-simple-int32-array">Exporting a simple <code class="docutils literal notranslate"><span class="pre">int32</span></code> array</a></li> |
| <li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#exporting-a-struct-float32-utf8-array">Exporting a <code class="docutils literal notranslate"><span class="pre">struct<float32,</span> <span class="pre">utf8></span></code> array</a></li> |
| </ul> |
| </li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-two-distinct-structures">Why two distinct structures?</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#updating-this-specification">Updating this specification</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#inspiration">Inspiration</a></li> |
| <li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#language-specific-protocols">Language-specific protocols</a></li> |
| </ul> |
| </nav></div> |
| |
| <div class="sidebar-secondary-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/main/docs/source/format/CDataInterface.rst"> |
| <i class="fa-solid fa-pencil"></i> |
| |
| |
| |
| Edit on GitHub |
| |
| |
| </a> |
| </div> |
| </div> |
| |
| </div></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script> |
| <script src="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2024 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
