Google Git
Sign in
apache / arrow-nanoarrow / refs/heads/asf-site / . / main / roadmap.html
blob: 3a3fd29911ee496683df00b77fcbf68efee89b1f [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" data-content_root="./" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Roadmap &#8212; nanoarrow documentation</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=8f2a1f02" />
<!-- So that users can add custom icons -->
<script src="_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="_static/documentation_options.js?v=5929fcd5"></script>
<script src="_static/doctools.js?v=9bcbadda"></script>
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'roadmap';</script>
<script>
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
DOCUMENTATION_OPTIONS.theme_switcher_json_url = 'https://arrow.apache.org/nanoarrow/switcher.json';
DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev';
DOCUMENTATION_OPTIONS.show_version_warning_banner =
false;
</script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="API Reference" href="reference/index.html" />
<link rel="prev" title="nanoarrow" href="getting-started/r.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="index.html">
<p class="title logo__title">nanoarrow documentation</p>
</a></div>
<div class="navbar-item">
<div class="version-switcher__container dropdown pst-js-only">
<button id="pst-version-switcher-button-2"
type="button"
class="version-switcher__button btn btn-sm dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-2"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-2"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-2">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="getting-started/index.html">
Getting Started
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="#">
Roadmap
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="reference/index.html">
API Reference
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="getting-started/index.html">
Getting Started
</a>
</li>
<li class="nav-item current active">
<a class="nav-link nav-internal" href="#">
Roadmap
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="reference/index.html">
API Reference
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<form class="bd-search d-flex align-items-center"
action="search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form></div>
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
aria-label="Section Navigation">
<p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
<div class="bd-toc-item navbar-nav"></div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Roadmap</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
--><section id="roadmap">
<h1>Roadmap<a class="headerlink" href="#roadmap" title="Link to this heading">#</a></h1>
<p>Apache Arrow nanoarrow is a relatively new library and is under active
development. Maintaining the balance between useful and minimal is
difficult to do; however, there are a number of features that fit
comforably within the scope of nanoarrow that have not yet been
scheduled for implementation.</p>
<section id="c-library">
<h2>C library<a class="headerlink" href="#c-library" title="Link to this heading">#</a></h2>
<ul class="simple">
<li><p><strong>Type coverage</strong>: The C library currently provides support for all
types that are available via the Arrow C Data interface except string
view/list view types. Support for these should be added in nanoarrow
as well
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/583">#583</a>,
<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/616">#616</a>,
<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/510">#510</a>).</p></li>
<li><p><strong>Remove Arrow C++ dependency for tests</strong>: The C library and IPC
extension rely on Arrow C++ for some test code that was written early
in the library’s development. These tests are valuable to ensure
compatibility between nanoarrow and Arrow C++; however, including
them in the default test suite complicates release verification for
some users and prevents testing in environments where Arrow C++ does
not currently build (e.g., WASM, compilers without C++17 support)
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/619">#619</a>).</p></li>
<li><p><strong>Test verbosity</strong>: Tests for the C library were written before
testing utilities in the <code class="docutils literal notranslate"><span class="pre">nanoarrow_testing</span></code> library were available
(and before there was a <code class="docutils literal notranslate"><span class="pre">nanoarrow_testing</span></code> library in which to put
new ones). As a result, some of them are very verbose and can be
difficult to read, which can and should be improved
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/577">#577</a>,
<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/566">#566</a>).</p></li>
<li><p><strong>C++ integration</strong>: The existing C++ integration is intentionally
minimal; however, there are likely improvements that could be made to
better integrate nanoarrow into existing C++ projects
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/599">#599</a>).</p></li>
<li><p><strong>Documentation</strong>: As the C library and its user base evolves,
documentation needs to be refined and expanded to support the current
set of use cases
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/187">#187</a>,
<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/497">#497</a>).</p></li>
<li><p><strong>IPC Dictionary support</strong>: The IPC extension does not currently
support reading dictionary messages an IPC stream
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/622">#622</a>).</p></li>
<li><p><strong>IPC Compression support</strong>: The IPC extension does not currently
support compressed streams using per-buffer compression, although
streams can be compressed outside the nanoarrow library (e.g., gzip
compression of the entire stream)
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/621">#621</a>)</p></li>
</ul>
</section>
<section id="r-bindings">
<h2>R bindings<a class="headerlink" href="#r-bindings" title="Link to this heading">#</a></h2>
<ul class="simple">
<li><p><strong>Conversion internals</strong>: The initial implementation of conversion
from Arrow data to R vectors was implemented in C and its verbosity
makes it difficult to add support for new types. The internals should
be refactored to make the conversion code easier to understand for
new developers
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/pull/392">#392</a>).</p></li>
<li><p><strong>Type support</strong>: The R bindings currently rely on the Arrow R
package for conversion of some R types (e.g., list_of), and some
types are not supported in nanoarrow nor the arrow R package (e.g.,
run-end encoding, list view, and string/binary view)
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/617">#617</a>).</p></li>
<li><p><strong>ALTREP support</strong>: A recent R release added enhanced ALTREP support
such that types that convert to <code class="docutils literal notranslate"><span class="pre">list()</span></code> can defer materialization
cost/allocation. Arrow sources that arrive in chunks (e.g., from a
<code class="docutils literal notranslate"><span class="pre">Table</span></code> or <code class="docutils literal notranslate"><span class="pre">ChunkedArray</span></code>) currently can’t be converted via any
ALTREP mechanism and support could be added
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/219">#219</a>).</p></li>
</ul>
</section>
<section id="python-bindings">
<h2>Python bindings<a class="headerlink" href="#python-bindings" title="Link to this heading">#</a></h2>
<ul class="simple">
<li><p><strong>Type support</strong>: The Python bindings do not currently support
unions, string/binary view, or list view, or run-end-encoded types.
When creating Arrow arrays from iterables of Python objects, some
types are not yet supported (e.g., struct, list, datetime objects)
(<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/618">#618</a>,
<a class="reference external" href="https://github.com/apache/arrow-nanoarrow/issues/620">#620</a>).</p></li>
</ul>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="getting-started/r.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">nanoarrow</p>
</div>
</a>
<a class="right-next"
href="reference/index.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">API Reference</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#c-library">C library</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#r-bindings">R bindings</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#python-bindings">Python bindings</a></li>
</ul>
</nav></div>
<div class="sidebar-secondary-item">
<div class="tocsection editthispage">
<a href="https://github.com/apache/arrow-nanoarrow/edit/main/docs/source/roadmap.rst">
<i class="fa-solid fa-pencil"></i>
Edit on GitHub
</a>
</div>
</div>
<div class="sidebar-secondary-item">
<div role="note" aria-label="source link">
<h3>This Page</h3>
<ul class="this-page-menu">
<li><a href="_sources/roadmap.rst.txt"
rel="nofollow">Show Source</a></li>
</ul>
</div></div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 2016-2025 Apache Software Foundation.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
<!-- # L10n: Setting the PST URL as an argument as this does not need to be localized -->
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.16.1.
</p></div>
</div>
</div>
</footer>
</body>
</html>