Google Git
Sign in
apache / arrow-site / asf-site / . / docs / dev / _modules / pyarrow / interchange / from_dataframe.html
blob: 9b950518236c04815221803e275fd5d43c5bf436 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" data-content_root="" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>pyarrow.interchange.from_dataframe &#8212; Apache Arrow v17.0.0.dev77</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
</script>
<!-- Loaded before other Sphinx assets -->
<link href="../../../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link href="../../../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link href="../../../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link href="../../../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../../../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../../../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../../../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" />
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css" />
<link rel="stylesheet" type="text/css" href="../../../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" />
<link rel="stylesheet" type="text/css" href="../../../_static/theme_overrides.css" />
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" />
<link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" />
<script src="../../../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script>
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
<script src="../../../_static/doctools.js"></script>
<script src="../../../_static/sphinx_highlight.js"></script>
<script src="../../../_static/clipboard.min.js"></script>
<script src="../../../_static/copybutton.js"></script>
<script src="../../../_static/design-tabs.js"></script>
<script>DOCUMENTATION_OPTIONS.pagename = '_modules/pyarrow/interchange/from_dataframe';</script>
<script>
DOCUMENTATION_OPTIONS.theme_version = '0.15.2';
DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json';
DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/';
DOCUMENTATION_OPTIONS.show_version_warning_banner = true;
</script>
<link rel="canonical" href="https://arrow.apache.org/docs/_modules/pyarrow/interchange/from_dataframe.html" />
<link rel="icon" href="../../../_static/favicon.ico"/>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
/* We explicitly disable cookie tracking to avoid privacy issues */
_paq.push(['disableCookies']);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '20']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>
Back to top
</button>
<input type="checkbox"
class="sidebar-toggle"
name="__primary"
id="__primary"/>
<label class="overlay overlay-primary" for="__primary"></label>
<input type="checkbox"
class="sidebar-toggle"
name="__secondary"
id="__secondary"/>
<label class="overlay overlay-secondary" for="__secondary"></label>
<div class="search-button__wrapper">
<div class="search-button__overlay"></div>
<div class="search-button__search-container">
<form class="bd-search d-flex align-items-center"
action="../../../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
id="search-input"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form></div>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar">
<div class="bd-header__inner bd-page-width">
<label class="sidebar-toggle primary-toggle" for="__primary">
<span class="fa-solid fa-bars"></span>
</label>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../../../index.html">
<img src="../../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev77 - Home"/>
<script>document.write(`<img src="../../../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev77 - Home"/>`);</script>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav class="navbar-nav">
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item">
<a class="nav-link nav-internal" href="../../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../../../developers/index.html">
Development
</a>
</li>
<li class="nav-item dropdown">
<button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links">
Implementations
</button>
<ul id="pst-nav-more-links" class="dropdown-menu">
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../c_glib/index.html">
C/GLib
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../cpp/index.html">
C++
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md">
C#
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17">
Go
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../java/index.html">
Java
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../js/index.html">
JavaScript
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/">
Julia
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md">
MATLAB
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/">
nanoarrow
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../python/index.html">
Python
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../r/index.html">
R
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md">
Ruby
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/">
Rust
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../status.html">
Implementation Status
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/">
C++ cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/">
Java cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/">
Python cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/">
R cookbook
</a>
</li>
</ul>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<script>
document.write(`
<button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
`);
</script>
</div>
<div class="navbar-item">
<script>
document.write(`
<div class="version-switcher__container dropdown">
<button id="pst-version-switcher-button-2"
type="button"
class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-2"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-2"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-2">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div>
`);
</script></div>
<div class="navbar-item">
<script>
document.write(`
<button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
</button>
`);
</script></div>
<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">X</span></a>
</li>
</ul></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<script>
document.write(`
<button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
`);
</script>
</div>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<div class="bd-sidebar-primary bd-sidebar hide-on-wide">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav class="navbar-nav">
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item">
<a class="nav-link nav-internal" href="../../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item">
<a class="nav-link nav-internal" href="../../../developers/index.html">
Development
</a>
</li>
<li class="nav-item dropdown">
<button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2">
Implementations
</button>
<ul id="pst-nav-more-links-2" class="dropdown-menu">
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../c_glib/index.html">
C/GLib
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../cpp/index.html">
C++
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md">
C#
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17">
Go
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../java/index.html">
Java
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../js/index.html">
JavaScript
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/">
Julia
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md">
MATLAB
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/">
nanoarrow
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../python/index.html">
Python
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../r/index.html">
R
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md">
Ruby
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/">
Rust
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-internal" href="../../../status.html">
Implementation Status
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/">
C++ cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/">
Java cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/">
Python cookbook
</a>
</li>
<li class="nav-item">
<a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/">
R cookbook
</a>
</li>
</ul>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item">
<script>
document.write(`
<div class="version-switcher__container dropdown">
<button id="pst-version-switcher-button-3"
type="button"
class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-3"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-3"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-3">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div>
`);
</script></div>
<div class="navbar-item">
<script>
document.write(`
<button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
<span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
<span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
</button>
`);
</script></div>
<div class="navbar-item"><ul class="navbar-icon-links navbar-nav"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span>
<span class="sr-only">X</span></a>
</li>
</ul></div>
</div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
</div>
<div id="rtd-footer-container"></div>
</div>
<main id="main-content" class="bd-main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../../../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../../index.html" class="nav-link">Module code</a></li>
<li class="breadcrumb-item"><a href="../../pyarrow.html" class="nav-link">pyarrow</a></li>
<li class="breadcrumb-item active" aria-current="page">pyarrow.inte...</li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<h1>Source code for pyarrow.interchange.from_dataframe</h1><div class="highlight"><pre>
<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
<span class="c1"># distributed with this work for additional information</span>
<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">annotations</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="p">(</span>
<span class="n">Any</span><span class="p">,</span>
<span class="n">Tuple</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span> <span class="nn">pyarrow.interchange.column</span> <span class="kn">import</span> <span class="p">(</span>
<span class="n">DtypeKind</span><span class="p">,</span>
<span class="n">ColumnBuffers</span><span class="p">,</span>
<span class="n">ColumnNullType</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="nn">pa</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">pyarrow.compute</span> <span class="k">as</span> <span class="nn">pc</span>
<span class="kn">from</span> <span class="nn">pyarrow.interchange.column</span> <span class="kn">import</span> <span class="n">Dtype</span>
<span class="c1"># A typing protocol could be added later to let Mypy validate code using</span>
<span class="c1"># `from_dataframe` better.</span>
<span class="n">DataFrameObject</span> <span class="o">=</span> <span class="n">Any</span>
<span class="n">ColumnObject</span> <span class="o">=</span> <span class="n">Any</span>
<span class="n">BufferObject</span> <span class="o">=</span> <span class="n">Any</span>
<span class="n">_PYARROW_DTYPES</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="n">DtypeKind</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">INT</span><span class="p">:</span> <span class="p">{</span><span class="mi">8</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">(),</span>
<span class="mi">16</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">int16</span><span class="p">(),</span>
<span class="mi">32</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">(),</span>
<span class="mi">64</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">()},</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">UINT</span><span class="p">:</span> <span class="p">{</span><span class="mi">8</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">uint8</span><span class="p">(),</span>
<span class="mi">16</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">uint16</span><span class="p">(),</span>
<span class="mi">32</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">uint32</span><span class="p">(),</span>
<span class="mi">64</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">uint64</span><span class="p">()},</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">FLOAT</span><span class="p">:</span> <span class="p">{</span><span class="mi">16</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">float16</span><span class="p">(),</span>
<span class="mi">32</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">float32</span><span class="p">(),</span>
<span class="mi">64</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">float64</span><span class="p">()},</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">BOOL</span><span class="p">:</span> <span class="p">{</span><span class="mi">1</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">(),</span>
<span class="mi">8</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">uint8</span><span class="p">()},</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">STRING</span><span class="p">:</span> <span class="p">{</span><span class="mi">8</span><span class="p">:</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">()},</span>
<span class="p">}</span>
<div class="viewcode-block" id="from_dataframe"><a class="viewcode-back" href="../../../python/generated/pyarrow.interchange.from_dataframe.html#pyarrow.interchange.from_dataframe">[docs]</a><span class="k">def</span> <span class="nf">from_dataframe</span><span class="p">(</span><span class="n">df</span><span class="p">:</span> <span class="n">DataFrameObject</span><span class="p">,</span> <span class="n">allow_copy</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Build a ``pa.Table`` from any DataFrame supporting the interchange protocol.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> df : DataFrameObject</span>
<span class="sd"> Object supporting the interchange protocol, i.e. `__dataframe__`</span>
<span class="sd"> method.</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.Table</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> &gt;&gt;&gt; import pyarrow</span>
<span class="sd"> &gt;&gt;&gt; from pyarrow.interchange import from_dataframe</span>
<span class="sd"> Convert a pandas dataframe to a pyarrow table:</span>
<span class="sd"> &gt;&gt;&gt; import pandas as pd</span>
<span class="sd"> &gt;&gt;&gt; df = pd.DataFrame({</span>
<span class="sd"> ... &quot;n_attendees&quot;: [100, 10, 1],</span>
<span class="sd"> ... &quot;country&quot;: [&quot;Italy&quot;, &quot;Spain&quot;, &quot;Slovenia&quot;],</span>
<span class="sd"> ... })</span>
<span class="sd"> &gt;&gt;&gt; df</span>
<span class="sd"> n_attendees country</span>
<span class="sd"> 0 100 Italy</span>
<span class="sd"> 1 10 Spain</span>
<span class="sd"> 2 1 Slovenia</span>
<span class="sd"> &gt;&gt;&gt; from_dataframe(df)</span>
<span class="sd"> pyarrow.Table</span>
<span class="sd"> n_attendees: int64</span>
<span class="sd"> country: large_string</span>
<span class="sd"> ----</span>
<span class="sd"> n_attendees: [[100,10,1]]</span>
<span class="sd"> country: [[&quot;Italy&quot;,&quot;Spain&quot;,&quot;Slovenia&quot;]]</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="p">):</span>
<span class="k">return</span> <span class="n">df</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="p">):</span>
<span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="o">.</span><span class="n">from_batches</span><span class="p">([</span><span class="n">df</span><span class="p">])</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="s2">&quot;__dataframe__&quot;</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;`df` does not support __dataframe__&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">_from_dataframe</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">__dataframe__</span><span class="p">(</span><span class="n">allow_copy</span><span class="o">=</span><span class="n">allow_copy</span><span class="p">),</span>
<span class="n">allow_copy</span><span class="o">=</span><span class="n">allow_copy</span><span class="p">)</span></div>
<span class="k">def</span> <span class="nf">_from_dataframe</span><span class="p">(</span><span class="n">df</span><span class="p">:</span> <span class="n">DataFrameObject</span><span class="p">,</span> <span class="n">allow_copy</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Build a ``pa.Table`` from the DataFrame interchange object.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> df : DataFrameObject</span>
<span class="sd"> Object supporting the interchange protocol, i.e. `__dataframe__`</span>
<span class="sd"> method.</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.Table</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">batches</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">get_chunks</span><span class="p">():</span>
<span class="n">batch</span> <span class="o">=</span> <span class="n">protocol_df_chunk_to_pyarrow</span><span class="p">(</span><span class="n">chunk</span><span class="p">,</span> <span class="n">allow_copy</span><span class="p">)</span>
<span class="n">batches</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">batches</span><span class="p">:</span>
<span class="n">batch</span> <span class="o">=</span> <span class="n">protocol_df_chunk_to_pyarrow</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<span class="n">batches</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span>
<span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">Table</span><span class="o">.</span><span class="n">from_batches</span><span class="p">(</span><span class="n">batches</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">protocol_df_chunk_to_pyarrow</span><span class="p">(</span>
<span class="n">df</span><span class="p">:</span> <span class="n">DataFrameObject</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Convert interchange protocol chunk to ``pa.RecordBatch``.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> df : DataFrameObject</span>
<span class="sd"> Object supporting the interchange protocol, i.e. `__dataframe__`</span>
<span class="sd"> method.</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.RecordBatch</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># We need a dict of columns here, with each column being a pa.Array</span>
<span class="n">columns</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">column_names</span><span class="p">():</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Column </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2"> is not a string&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Column </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2"> is not unique&quot;</span><span class="p">)</span>
<span class="n">col</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">get_column_by_name</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="n">dtype</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">dtype</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">dtype</span> <span class="ow">in</span> <span class="p">(</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">INT</span><span class="p">,</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">UINT</span><span class="p">,</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">FLOAT</span><span class="p">,</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">STRING</span><span class="p">,</span>
<span class="n">DtypeKind</span><span class="o">.</span><span class="n">DATETIME</span><span class="p">,</span>
<span class="p">):</span>
<span class="n">columns</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">column_to_array</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">allow_copy</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">dtype</span> <span class="o">==</span> <span class="n">DtypeKind</span><span class="o">.</span><span class="n">BOOL</span><span class="p">:</span>
<span class="n">columns</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">bool_column_to_array</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">allow_copy</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">dtype</span> <span class="o">==</span> <span class="n">DtypeKind</span><span class="o">.</span><span class="n">CATEGORICAL</span><span class="p">:</span>
<span class="n">columns</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">categorical_column_to_dictionary</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">allow_copy</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Data type </span><span class="si">{</span><span class="n">dtype</span><span class="si">}</span><span class="s2"> not handled yet&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">column_to_array</span><span class="p">(</span>
<span class="n">col</span><span class="p">:</span> <span class="n">ColumnObject</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Convert a column holding one of the primitive dtypes to a PyArrow array.</span>
<span class="sd"> A primitive type is one of: int, uint, float, bool (1 bit).</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> col : ColumnObject</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.Array</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">buffers</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">get_buffers</span><span class="p">()</span>
<span class="n">data_type</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">dtype</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">buffers_to_array</span><span class="p">(</span><span class="n">buffers</span><span class="p">,</span> <span class="n">data_type</span><span class="p">,</span>
<span class="n">col</span><span class="o">.</span><span class="n">size</span><span class="p">(),</span>
<span class="n">col</span><span class="o">.</span><span class="n">describe_null</span><span class="p">,</span>
<span class="n">col</span><span class="o">.</span><span class="n">offset</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">)</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">def</span> <span class="nf">bool_column_to_array</span><span class="p">(</span>
<span class="n">col</span><span class="p">:</span> <span class="n">ColumnObject</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Convert a column holding boolean dtype to a PyArrow array.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> col : ColumnObject</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.Array</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">buffers</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">get_buffers</span><span class="p">()</span>
<span class="n">size</span> <span class="o">=</span> <span class="n">buffers</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">][</span><span class="mi">1</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
<span class="c1"># If booleans are byte-packed a copy to bit-packed will be made</span>
<span class="k">if</span> <span class="n">size</span> <span class="o">==</span> <span class="mi">8</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">allow_copy</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
<span class="s2">&quot;Boolean column will be casted from uint8 and a copy &quot;</span>
<span class="s2">&quot;is required which is forbidden by allow_copy=False&quot;</span>
<span class="p">)</span>
<span class="n">data_type</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">dtype</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">buffers_to_array</span><span class="p">(</span><span class="n">buffers</span><span class="p">,</span> <span class="n">data_type</span><span class="p">,</span>
<span class="n">col</span><span class="o">.</span><span class="n">size</span><span class="p">(),</span>
<span class="n">col</span><span class="o">.</span><span class="n">describe_null</span><span class="p">,</span>
<span class="n">col</span><span class="o">.</span><span class="n">offset</span><span class="p">)</span>
<span class="k">if</span> <span class="n">size</span> <span class="o">==</span> <span class="mi">8</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">())</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">def</span> <span class="nf">categorical_column_to_dictionary</span><span class="p">(</span>
<span class="n">col</span><span class="p">:</span> <span class="n">ColumnObject</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">DictionaryArray</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Convert a column holding categorical data to a pa.DictionaryArray.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> col : ColumnObject</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.DictionaryArray</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">allow_copy</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
<span class="s2">&quot;Categorical column will be casted from uint8 and a copy &quot;</span>
<span class="s2">&quot;is required which is forbidden by allow_copy=False&quot;</span>
<span class="p">)</span>
<span class="n">categorical</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">describe_categorical</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">categorical</span><span class="p">[</span><span class="s2">&quot;is_dictionary&quot;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
<span class="s2">&quot;Non-dictionary categoricals not supported yet&quot;</span><span class="p">)</span>
<span class="c1"># We need to first convert the dictionary column</span>
<span class="n">cat_column</span> <span class="o">=</span> <span class="n">categorical</span><span class="p">[</span><span class="s2">&quot;categories&quot;</span><span class="p">]</span>
<span class="n">dictionary</span> <span class="o">=</span> <span class="n">column_to_array</span><span class="p">(</span><span class="n">cat_column</span><span class="p">)</span>
<span class="c1"># Then we need to convert the indices</span>
<span class="c1"># Here we need to use the buffer data type!</span>
<span class="n">buffers</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">get_buffers</span><span class="p">()</span>
<span class="n">_</span><span class="p">,</span> <span class="n">data_type</span> <span class="o">=</span> <span class="n">buffers</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">]</span>
<span class="n">indices</span> <span class="o">=</span> <span class="n">buffers_to_array</span><span class="p">(</span><span class="n">buffers</span><span class="p">,</span> <span class="n">data_type</span><span class="p">,</span>
<span class="n">col</span><span class="o">.</span><span class="n">size</span><span class="p">(),</span>
<span class="n">col</span><span class="o">.</span><span class="n">describe_null</span><span class="p">,</span>
<span class="n">col</span><span class="o">.</span><span class="n">offset</span><span class="p">)</span>
<span class="c1"># Constructing a pa.DictionaryArray</span>
<span class="n">dict_array</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">DictionaryArray</span><span class="o">.</span><span class="n">from_arrays</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">dictionary</span><span class="p">)</span>
<span class="k">return</span> <span class="n">dict_array</span>
<span class="k">def</span> <span class="nf">parse_datetime_format_str</span><span class="p">(</span><span class="n">format_str</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse datetime `format_str` to interpret the `data`.&quot;&quot;&quot;</span>
<span class="c1"># timestamp &#39;ts{unit}:tz&#39;</span>
<span class="n">timestamp_meta</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;ts([smun]):(.*)&quot;</span><span class="p">,</span> <span class="n">format_str</span><span class="p">)</span>
<span class="k">if</span> <span class="n">timestamp_meta</span><span class="p">:</span>
<span class="n">unit</span><span class="p">,</span> <span class="n">tz</span> <span class="o">=</span> <span class="n">timestamp_meta</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">timestamp_meta</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="k">if</span> <span class="n">unit</span> <span class="o">!=</span> <span class="s2">&quot;s&quot;</span><span class="p">:</span>
<span class="c1"># the format string describes only a first letter of the unit, so</span>
<span class="c1"># add one extra letter to convert the unit to numpy-style:</span>
<span class="c1"># &#39;m&#39; -&gt; &#39;ms&#39;, &#39;u&#39; -&gt; &#39;us&#39;, &#39;n&#39; -&gt; &#39;ns&#39;</span>
<span class="n">unit</span> <span class="o">+=</span> <span class="s2">&quot;s&quot;</span>
<span class="k">return</span> <span class="n">unit</span><span class="p">,</span> <span class="n">tz</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;DateTime kind is not supported: </span><span class="si">{</span><span class="n">format_str</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">map_date_type</span><span class="p">(</span><span class="n">data_type</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Map column date type to pyarrow date type. &quot;&quot;&quot;</span>
<span class="n">kind</span><span class="p">,</span> <span class="n">bit_width</span><span class="p">,</span> <span class="n">f_string</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">data_type</span>
<span class="k">if</span> <span class="n">kind</span> <span class="o">==</span> <span class="n">DtypeKind</span><span class="o">.</span><span class="n">DATETIME</span><span class="p">:</span>
<span class="n">unit</span><span class="p">,</span> <span class="n">tz</span> <span class="o">=</span> <span class="n">parse_datetime_format_str</span><span class="p">(</span><span class="n">f_string</span><span class="p">)</span>
<span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="n">unit</span><span class="p">,</span> <span class="n">tz</span><span class="o">=</span><span class="n">tz</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">pa_dtype</span> <span class="o">=</span> <span class="n">_PYARROW_DTYPES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">kind</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">bit_width</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="c1"># Error if dtype is not supported</span>
<span class="k">if</span> <span class="n">pa_dtype</span><span class="p">:</span>
<span class="k">return</span> <span class="n">pa_dtype</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;Conversion for </span><span class="si">{</span><span class="n">data_type</span><span class="si">}</span><span class="s2"> is not yet supported.&quot;</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">buffers_to_array</span><span class="p">(</span>
<span class="n">buffers</span><span class="p">:</span> <span class="n">ColumnBuffers</span><span class="p">,</span>
<span class="n">data_type</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">DtypeKind</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span>
<span class="n">length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
<span class="n">describe_null</span><span class="p">:</span> <span class="n">ColumnNullType</span><span class="p">,</span>
<span class="n">offset</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Build a PyArrow array from the passed buffer.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> buffer : ColumnBuffers</span>
<span class="sd"> Dictionary containing tuples of underlying buffers and</span>
<span class="sd"> their associated dtype.</span>
<span class="sd"> data_type : Tuple[DtypeKind, int, str, str],</span>
<span class="sd"> Dtype description of the column as a tuple ``(kind, bit-width, format string,</span>
<span class="sd"> endianness)``.</span>
<span class="sd"> length : int</span>
<span class="sd"> The number of values in the array.</span>
<span class="sd"> describe_null: ColumnNullType</span>
<span class="sd"> Null representation the column dtype uses,</span>
<span class="sd"> as a tuple ``(kind, value)``</span>
<span class="sd"> offset : int, default: 0</span>
<span class="sd"> Number of elements to offset from the start of the buffer.</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.Array</span>
<span class="sd"> Notes</span>
<span class="sd"> -----</span>
<span class="sd"> The returned array doesn&#39;t own the memory. The caller of this function</span>
<span class="sd"> is responsible for keeping the memory owner object alive as long as</span>
<span class="sd"> the returned PyArrow array is being used.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">data_buff</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">buffers</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">validity_buff</span><span class="p">,</span> <span class="n">validity_dtype</span> <span class="o">=</span> <span class="n">buffers</span><span class="p">[</span><span class="s2">&quot;validity&quot;</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
<span class="n">validity_buff</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">offset_buff</span><span class="p">,</span> <span class="n">offset_dtype</span> <span class="o">=</span> <span class="n">buffers</span><span class="p">[</span><span class="s2">&quot;offsets&quot;</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
<span class="n">offset_buff</span> <span class="o">=</span> <span class="kc">None</span>
<span class="c1"># Construct a pyarrow Buffer</span>
<span class="n">data_pa_buffer</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">foreign_buffer</span><span class="p">(</span><span class="n">data_buff</span><span class="o">.</span><span class="n">ptr</span><span class="p">,</span> <span class="n">data_buff</span><span class="o">.</span><span class="n">bufsize</span><span class="p">,</span>
<span class="n">base</span><span class="o">=</span><span class="n">data_buff</span><span class="p">)</span>
<span class="c1"># Construct a validity pyarrow Buffer, if applicable</span>
<span class="k">if</span> <span class="n">validity_buff</span><span class="p">:</span>
<span class="n">validity_pa_buff</span> <span class="o">=</span> <span class="n">validity_buffer_from_mask</span><span class="p">(</span><span class="n">validity_buff</span><span class="p">,</span>
<span class="n">validity_dtype</span><span class="p">,</span>
<span class="n">describe_null</span><span class="p">,</span>
<span class="n">length</span><span class="p">,</span>
<span class="n">offset</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">validity_pa_buff</span> <span class="o">=</span> <span class="n">validity_buffer_nan_sentinel</span><span class="p">(</span><span class="n">data_pa_buffer</span><span class="p">,</span>
<span class="n">data_type</span><span class="p">,</span>
<span class="n">describe_null</span><span class="p">,</span>
<span class="n">length</span><span class="p">,</span>
<span class="n">offset</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">)</span>
<span class="c1"># Construct a pyarrow Array from buffers</span>
<span class="n">data_dtype</span> <span class="o">=</span> <span class="n">map_date_type</span><span class="p">(</span><span class="n">data_type</span><span class="p">)</span>
<span class="k">if</span> <span class="n">offset_buff</span><span class="p">:</span>
<span class="n">_</span><span class="p">,</span> <span class="n">offset_bit_width</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">offset_dtype</span>
<span class="c1"># If an offset buffer exists, construct an offset pyarrow Buffer</span>
<span class="c1"># and add it to the construction of an array</span>
<span class="n">offset_pa_buffer</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">foreign_buffer</span><span class="p">(</span><span class="n">offset_buff</span><span class="o">.</span><span class="n">ptr</span><span class="p">,</span>
<span class="n">offset_buff</span><span class="o">.</span><span class="n">bufsize</span><span class="p">,</span>
<span class="n">base</span><span class="o">=</span><span class="n">offset_buff</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data_type</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;U&#39;</span><span class="p">:</span>
<span class="n">string_type</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">large_string</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="n">offset_bit_width</span> <span class="o">==</span> <span class="mi">64</span><span class="p">:</span>
<span class="n">string_type</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">large_string</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">string_type</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">()</span>
<span class="n">array</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="o">.</span><span class="n">from_buffers</span><span class="p">(</span>
<span class="n">string_type</span><span class="p">,</span>
<span class="n">length</span><span class="p">,</span>
<span class="p">[</span><span class="n">validity_pa_buff</span><span class="p">,</span> <span class="n">offset_pa_buffer</span><span class="p">,</span> <span class="n">data_pa_buffer</span><span class="p">],</span>
<span class="n">offset</span><span class="o">=</span><span class="n">offset</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">array</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="o">.</span><span class="n">from_buffers</span><span class="p">(</span>
<span class="n">data_dtype</span><span class="p">,</span>
<span class="n">length</span><span class="p">,</span>
<span class="p">[</span><span class="n">validity_pa_buff</span><span class="p">,</span> <span class="n">data_pa_buffer</span><span class="p">],</span>
<span class="n">offset</span><span class="o">=</span><span class="n">offset</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">array</span>
<span class="k">def</span> <span class="nf">validity_buffer_from_mask</span><span class="p">(</span>
<span class="n">validity_buff</span><span class="p">:</span> <span class="n">BufferObject</span><span class="p">,</span>
<span class="n">validity_dtype</span><span class="p">:</span> <span class="n">Dtype</span><span class="p">,</span>
<span class="n">describe_null</span><span class="p">:</span> <span class="n">ColumnNullType</span><span class="p">,</span>
<span class="n">length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
<span class="n">offset</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">Buffer</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Build a PyArrow buffer from the passed mask buffer.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> validity_buff : BufferObject</span>
<span class="sd"> Tuple of underlying validity buffer and associated dtype.</span>
<span class="sd"> validity_dtype : Dtype</span>
<span class="sd"> Dtype description as a tuple ``(kind, bit-width, format string,</span>
<span class="sd"> endianness)``.</span>
<span class="sd"> describe_null : ColumnNullType</span>
<span class="sd"> Null representation the column dtype uses,</span>
<span class="sd"> as a tuple ``(kind, value)``</span>
<span class="sd"> length : int</span>
<span class="sd"> The number of values in the array.</span>
<span class="sd"> offset : int, default: 0</span>
<span class="sd"> Number of elements to offset from the start of the buffer.</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.Buffer</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">null_kind</span><span class="p">,</span> <span class="n">sentinel_val</span> <span class="o">=</span> <span class="n">describe_null</span>
<span class="n">validity_kind</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">validity_dtype</span>
<span class="k">assert</span> <span class="n">validity_kind</span> <span class="o">==</span> <span class="n">DtypeKind</span><span class="o">.</span><span class="n">BOOL</span>
<span class="k">if</span> <span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">NON_NULLABLE</span><span class="p">:</span>
<span class="c1"># Sliced array can have a NON_NULLABLE ColumnNullType due</span>
<span class="c1"># to no missing values in that slice of an array though the bitmask</span>
<span class="c1"># exists and validity_buff must be set to None in this case</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">elif</span> <span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">USE_BYTEMASK</span> <span class="ow">or</span> <span class="p">(</span>
<span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">USE_BITMASK</span> <span class="ow">and</span> <span class="n">sentinel_val</span> <span class="o">==</span> <span class="mi">1</span>
<span class="p">):</span>
<span class="n">buff</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">foreign_buffer</span><span class="p">(</span><span class="n">validity_buff</span><span class="o">.</span><span class="n">ptr</span><span class="p">,</span>
<span class="n">validity_buff</span><span class="o">.</span><span class="n">bufsize</span><span class="p">,</span>
<span class="n">base</span><span class="o">=</span><span class="n">validity_buff</span><span class="p">)</span>
<span class="k">if</span> <span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">USE_BYTEMASK</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">allow_copy</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
<span class="s2">&quot;To create a bitmask a copy of the data is &quot;</span>
<span class="s2">&quot;required which is forbidden by allow_copy=False&quot;</span>
<span class="p">)</span>
<span class="n">mask</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="o">.</span><span class="n">from_buffers</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">int8</span><span class="p">(),</span> <span class="n">length</span><span class="p">,</span>
<span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">buff</span><span class="p">],</span>
<span class="n">offset</span><span class="o">=</span><span class="n">offset</span><span class="p">)</span>
<span class="n">mask_bool</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">mask</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">mask_bool</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="o">.</span><span class="n">from_buffers</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">bool_</span><span class="p">(),</span> <span class="n">length</span><span class="p">,</span>
<span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">buff</span><span class="p">],</span>
<span class="n">offset</span><span class="o">=</span><span class="n">offset</span><span class="p">)</span>
<span class="k">if</span> <span class="n">sentinel_val</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">mask_bool</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">invert</span><span class="p">(</span><span class="n">mask_bool</span><span class="p">)</span>
<span class="k">return</span> <span class="n">mask_bool</span><span class="o">.</span><span class="n">buffers</span><span class="p">()[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">elif</span> <span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">USE_BITMASK</span> <span class="ow">and</span> <span class="n">sentinel_val</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="n">pa</span><span class="o">.</span><span class="n">foreign_buffer</span><span class="p">(</span><span class="n">validity_buff</span><span class="o">.</span><span class="n">ptr</span><span class="p">,</span>
<span class="n">validity_buff</span><span class="o">.</span><span class="n">bufsize</span><span class="p">,</span>
<span class="n">base</span><span class="o">=</span><span class="n">validity_buff</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">describe_null</span><span class="si">}</span><span class="s2"> null representation is not yet supported.&quot;</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">validity_buffer_nan_sentinel</span><span class="p">(</span>
<span class="n">data_pa_buffer</span><span class="p">:</span> <span class="n">BufferObject</span><span class="p">,</span>
<span class="n">data_type</span><span class="p">:</span> <span class="n">Dtype</span><span class="p">,</span>
<span class="n">describe_null</span><span class="p">:</span> <span class="n">ColumnNullType</span><span class="p">,</span>
<span class="n">length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
<span class="n">offset</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
<span class="n">allow_copy</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pa</span><span class="o">.</span><span class="n">Buffer</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Build a PyArrow buffer from NaN or sentinel values.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> data_pa_buffer : pa.Buffer</span>
<span class="sd"> PyArrow buffer for the column data.</span>
<span class="sd"> data_type : Dtype</span>
<span class="sd"> Dtype description as a tuple ``(kind, bit-width, format string,</span>
<span class="sd"> endianness)``.</span>
<span class="sd"> describe_null : ColumnNullType</span>
<span class="sd"> Null representation the column dtype uses,</span>
<span class="sd"> as a tuple ``(kind, value)``</span>
<span class="sd"> length : int</span>
<span class="sd"> The number of values in the array.</span>
<span class="sd"> offset : int, default: 0</span>
<span class="sd"> Number of elements to offset from the start of the buffer.</span>
<span class="sd"> allow_copy : bool, default: True</span>
<span class="sd"> Whether to allow copying the memory to perform the conversion</span>
<span class="sd"> (if false then zero-copy approach is requested).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pa.Buffer</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">kind</span><span class="p">,</span> <span class="n">bit_width</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">data_type</span>
<span class="n">data_dtype</span> <span class="o">=</span> <span class="n">map_date_type</span><span class="p">(</span><span class="n">data_type</span><span class="p">)</span>
<span class="n">null_kind</span><span class="p">,</span> <span class="n">sentinel_val</span> <span class="o">=</span> <span class="n">describe_null</span>
<span class="c1"># Check for float NaN values</span>
<span class="k">if</span> <span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">USE_NAN</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">allow_copy</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
<span class="s2">&quot;To create a bitmask a copy of the data is &quot;</span>
<span class="s2">&quot;required which is forbidden by allow_copy=False&quot;</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">kind</span> <span class="o">==</span> <span class="n">DtypeKind</span><span class="o">.</span><span class="n">FLOAT</span> <span class="ow">and</span> <span class="n">bit_width</span> <span class="o">==</span> <span class="mi">16</span><span class="p">:</span>
<span class="c1"># &#39;pyarrow.compute.is_nan&#39; kernel not yet implemented</span>
<span class="c1"># for float16</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">data_type</span><span class="si">}</span><span class="s2"> with </span><span class="si">{</span><span class="n">null_kind</span><span class="si">}</span><span class="s2"> is not yet supported.&quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">pyarrow_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="o">.</span><span class="n">from_buffers</span><span class="p">(</span>
<span class="n">data_dtype</span><span class="p">,</span>
<span class="n">length</span><span class="p">,</span>
<span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">data_pa_buffer</span><span class="p">],</span>
<span class="n">offset</span><span class="o">=</span><span class="n">offset</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">mask</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">is_nan</span><span class="p">(</span><span class="n">pyarrow_data</span><span class="p">)</span>
<span class="n">mask</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">invert</span><span class="p">(</span><span class="n">mask</span><span class="p">)</span>
<span class="k">return</span> <span class="n">mask</span><span class="o">.</span><span class="n">buffers</span><span class="p">()[</span><span class="mi">1</span><span class="p">]</span>
<span class="c1"># Check for sentinel values</span>
<span class="k">elif</span> <span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">USE_SENTINEL</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">allow_copy</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
<span class="s2">&quot;To create a bitmask a copy of the data is &quot;</span>
<span class="s2">&quot;required which is forbidden by allow_copy=False&quot;</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">kind</span> <span class="o">==</span> <span class="n">DtypeKind</span><span class="o">.</span><span class="n">DATETIME</span><span class="p">:</span>
<span class="n">sentinel_dtype</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">sentinel_dtype</span> <span class="o">=</span> <span class="n">data_dtype</span>
<span class="n">pyarrow_data</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">Array</span><span class="o">.</span><span class="n">from_buffers</span><span class="p">(</span><span class="n">sentinel_dtype</span><span class="p">,</span>
<span class="n">length</span><span class="p">,</span>
<span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">data_pa_buffer</span><span class="p">],</span>
<span class="n">offset</span><span class="o">=</span><span class="n">offset</span><span class="p">)</span>
<span class="n">sentinel_arr</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">equal</span><span class="p">(</span><span class="n">pyarrow_data</span><span class="p">,</span> <span class="n">sentinel_val</span><span class="p">)</span>
<span class="n">mask_bool</span> <span class="o">=</span> <span class="n">pc</span><span class="o">.</span><span class="n">invert</span><span class="p">(</span><span class="n">sentinel_arr</span><span class="p">)</span>
<span class="k">return</span> <span class="n">mask_bool</span><span class="o">.</span><span class="n">buffers</span><span class="p">()[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">elif</span> <span class="n">null_kind</span> <span class="o">==</span> <span class="n">ColumnNullType</span><span class="o">.</span><span class="n">NON_NULLABLE</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">describe_null</span><span class="si">}</span><span class="s2"> null representation is not yet supported.&quot;</span><span class="p">)</span>
</pre></div>
</article>
<footer class="prev-next-footer">
<div class="prev-next-area">
</div>
</footer>
</div>
<div class="bd-sidebar-secondary bd-toc"></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script src="../../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script>
<script src="../../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 2016-2024 Apache Software Foundation.
Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2.
</p></div>
</div>
</div>
</footer>
</body>
</html>