Google Git
Sign in
apache / arrow-site / 1e645257273c2521a371004aa4d87af19a1c47bc / . / docs / dev / _modules / pyarrow / feather.html
blob: 147cdd8279469abe9de8392a469f060b485627bb [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" data-content_root="../../" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>pyarrow.feather &#8212; Apache Arrow v23.0.0.dev37</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=03e43079" />
<link rel="stylesheet" type="text/css" href="../../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../../_static/sphinx-design.min.css?v=95c83b7e" />
<link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css?v=8dcd28dc" />
<!-- So that users can add custom icons -->
<script src="../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../../_static/documentation_options.js?v=9fc6757a"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../../_static/copybutton.js?v=3bb21c8c"></script>
<script src="../../_static/design-tabs.js?v=f930bc37"></script>
<script type="module" src="https://cdn.jsdelivr.net/npm/mermaid@11.2.0/dist/mermaid.esm.min.mjs"></script>
<script type="module" src="https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk@0.1.4/dist/mermaid-layout-elk.esm.min.mjs"></script>
<script type="module">import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@11.2.0/dist/mermaid.esm.min.mjs";import elkLayouts from "https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk@0.1.4/dist/mermaid-layout-elk.esm.min.mjs";mermaid.registerLayoutLoaders(elkLayouts);mermaid.initialize({startOnLoad:false});</script>
<script type="module">
import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@11.2.0/dist/mermaid.esm.min.mjs";
window.addEventListener("load", () => mermaid.run());
</script>
<script>DOCUMENTATION_OPTIONS.pagename = '_modules/pyarrow/feather';</script>
<script>
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json';
DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/';
DOCUMENTATION_OPTIONS.show_version_warning_banner =
true;
</script>
<link rel="canonical" href="https://arrow.apache.org/docs/_modules/pyarrow/feather.html" />
<link rel="icon" href="../../_static/favicon.ico"/>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="23.0.0.dev37" />
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
/* We explicitly disable cookie tracking to avoid privacy issues */
_paq.push(['disableCookies']);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '20']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class=" navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../../index.html">
<img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v23.0.0.dev37 - Home"/>
<img src="../../_static/arrow-dark.png" class="logo__image only-dark pst-js-only" alt="Apache Arrow v23.0.0.dev37 - Home"/>
</a></div>
</div>
<div class=" navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../developers/index.html">
Development
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../implementations.html">
Implementations
</a>
</li>
</ul>
</nav></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item"><div class="kapa-ai-bot">
<script
async
src="https://widget.kapa.ai/kapa-widget.bundle.js"
data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2"
data-project-name="Apache Arrow"
data-project-color="#000000"
data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png"
data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc."
data-consent-required="true"
data-user-analytics-cookie-enabled="false"
data-consent-screen-disclaimer="By clicking &quot;I agree, let's chat&quot;, you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies."
></script>
</div>
</div>
<div class="navbar-item">
<div class="version-switcher__container dropdown pst-js-only">
<button id="pst-version-switcher-button-2"
type="button"
class="version-switcher__button btn btn-sm dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-2"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-2"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-2">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i>
<span class="sr-only">LinkedIn</span></a>
</li>
<li class="nav-item">
<a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i>
<span class="sr-only">BlueSky</span></a>
</li>
</ul></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar hide-on-wide">
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<nav>
<ul class="bd-navbar-elements navbar-nav">
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../format/index.html">
Specifications
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../developers/index.html">
Development
</a>
</li>
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../implementations.html">
Implementations
</a>
</li>
</ul>
</nav></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item"><div class="kapa-ai-bot">
<script
async
src="https://widget.kapa.ai/kapa-widget.bundle.js"
data-website-id="9db461d5-ac77-4b3f-a5c5-75efa78339d2"
data-project-name="Apache Arrow"
data-project-color="#000000"
data-project-logo="https://arrow.apache.org/img/arrow-logo_chevrons_white-txt_black-bg.png"
data-modal-disclaimer="This is a custom LLM with access to all [Arrow documentation](https://arrow.apache.org/docs/). Please include the language you are using in your question, e.g., Python, C++, Java, R, etc."
data-consent-required="true"
data-user-analytics-cookie-enabled="false"
data-consent-screen-disclaimer="By clicking &quot;I agree, let's chat&quot;, you consent to the use of the AI assistant in accordance with kapa.ai's [Privacy Policy](https://www.kapa.ai/content/privacy-policy). This service uses reCAPTCHA, which requires your consent to Google's [Privacy Policy](https://policies.google.com/privacy) and [Terms of Service](https://policies.google.com/terms). By proceeding, you explicitly agree to both kapa.ai's and Google's privacy policies."
></script>
</div>
</div>
<div class="navbar-item">
<div class="version-switcher__container dropdown pst-js-only">
<button id="pst-version-switcher-button-3"
type="button"
class="version-switcher__button btn btn-sm dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-3"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-3"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-3">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div></div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
<div class="navbar-item"><ul class="navbar-icon-links"
aria-label="Icon Links">
<li class="nav-item">
<a href="https://github.com/apache/arrow" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i>
<span class="sr-only">GitHub</span></a>
</li>
<li class="nav-item">
<a href="https://www.linkedin.com/company/apache-arrow/" title="LinkedIn" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-linkedin fa-lg" aria-hidden="true"></i>
<span class="sr-only">LinkedIn</span></a>
</li>
<li class="nav-item">
<a href="https://bsky.app/profile/arrow.apache.org" title="BlueSky" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-bluesky fa-lg" aria-hidden="true"></i>
<span class="sr-only">BlueSky</span></a>
</li>
</ul></div>
</div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
<div class="sidebar-primary-item">
<div id="ethical-ad-placement"
class="flat"
data-ea-publisher="readthedocs"
data-ea-type="readthedocs-sidebar"
data-ea-manual="true">
</div></div>
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item"><a href="../index.html" class="nav-link">Module code</a></li>
<li class="breadcrumb-item"><a href="../pyarrow.html" class="nav-link">pyarrow</a></li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">pyarrow.feather</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<h1>Source code for pyarrow.feather</h1><div class="highlight"><pre>
<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
<span class="c1"># distributed with this work for additional information</span>
<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Sequence</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.pandas_compat</span><span class="w"> </span><span class="kn">import</span> <span class="n">_pandas_api</span> <span class="c1"># noqa</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">Codec</span><span class="p">,</span> <span class="n">Table</span><span class="p">,</span> <span class="c1"># noqa</span>
<span class="n">concat_tables</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow.lib</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">ext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="kn">import</span> <span class="n">_feather</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pyarrow._feather</span><span class="w"> </span><span class="kn">import</span> <span class="n">FeatherError</span> <span class="c1"># noqa: F401</span>
<span class="k">class</span><span class="w"> </span><span class="nc">FeatherDataset</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Encapsulates details of reading a list of Feather files.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> path_or_paths : List[str]</span>
<span class="sd"> A list of file names</span>
<span class="sd"> validate_schema : bool, default True</span>
<span class="sd"> Check that individual file schemas are all the same / compatible</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path_or_paths</span><span class="p">,</span> <span class="n">validate_schema</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">paths</span> <span class="o">=</span> <span class="n">path_or_paths</span>
<span class="bp">self</span><span class="o">.</span><span class="n">validate_schema</span> <span class="o">=</span> <span class="n">validate_schema</span>
<span class="k">def</span><span class="w"> </span><span class="nf">read_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Read multiple feather files as a single pyarrow.Table</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> columns : List[str]</span>
<span class="sd"> Names of columns to read from the file</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pyarrow.Table</span>
<span class="sd"> Content of the file as a table (of columns)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">_fil</span> <span class="o">=</span> <span class="n">read_table</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">paths</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_tables</span> <span class="o">=</span> <span class="p">[</span><span class="n">_fil</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">_fil</span><span class="o">.</span><span class="n">schema</span>
<span class="k">for</span> <span class="n">path</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">paths</span><span class="p">[</span><span class="mi">1</span><span class="p">:]:</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">read_table</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">validate_schema</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">validate_schemas</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_tables</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">table</span><span class="p">)</span>
<span class="k">return</span> <span class="n">concat_tables</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_tables</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">validate_schemas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">piece</span><span class="p">,</span> <span class="n">table</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">equals</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Schema in </span><span class="si">{</span><span class="n">piece</span><span class="si">}</span><span class="s1"> was different. </span><span class="se">\n</span><span class="s1">&#39;</span>
<span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="si">}</span><span class="se">\n\n</span><span class="s1">vs</span><span class="se">\n\n</span><span class="si">{</span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">read_pandas</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Read multiple Parquet files as a single pandas DataFrame</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> columns : List[str]</span>
<span class="sd"> Names of columns to read from the file</span>
<span class="sd"> use_threads : bool, default True</span>
<span class="sd"> Use multiple threads when converting to pandas</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> pandas.DataFrame</span>
<span class="sd"> Content of the file as a pandas DataFrame (of columns)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">read_table</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">)</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span>
<span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">check_chunked_overflow</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">col</span><span class="p">):</span>
<span class="k">if</span> <span class="n">col</span><span class="o">.</span><span class="n">num_chunks</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span>
<span class="k">if</span> <span class="n">col</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">binary</span><span class="p">(),</span> <span class="n">ext</span><span class="o">.</span><span class="n">string</span><span class="p">()):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Column &#39;</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">&#39; exceeds 2GB maximum capacity of &quot;</span>
<span class="s2">&quot;a Feather binary column. This restriction may be &quot;</span>
<span class="s2">&quot;lifted in the future&quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># TODO(wesm): Not sure when else this might be reached</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;Column &#39;</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">&#39; of type </span><span class="si">{</span><span class="n">col</span><span class="o">.</span><span class="n">type</span><span class="si">}</span><span class="s2"> was chunked on conversion to Arrow &quot;</span>
<span class="s2">&quot;and cannot be currently written to Feather format&quot;</span>
<span class="p">)</span>
<span class="n">_FEATHER_SUPPORTED_CODECS</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;lz4&#39;</span><span class="p">,</span> <span class="s1">&#39;zstd&#39;</span><span class="p">,</span> <span class="s1">&#39;uncompressed&#39;</span><span class="p">}</span>
<div class="viewcode-block" id="write_feather">
<a class="viewcode-back" href="../../python/generated/pyarrow.feather.write_feather.html#pyarrow.feather.write_feather">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">write_feather</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">compression</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">compression_level</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">chunksize</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Write a pandas.DataFrame to Feather format.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> df : pandas.DataFrame or pyarrow.Table</span>
<span class="sd"> Data to write out as Feather format.</span>
<span class="sd"> dest : str</span>
<span class="sd"> Local destination path.</span>
<span class="sd"> compression : string, default None</span>
<span class="sd"> Can be one of {&quot;zstd&quot;, &quot;lz4&quot;, &quot;uncompressed&quot;}. The default of None uses</span>
<span class="sd"> LZ4 for V2 files if it is available, otherwise uncompressed.</span>
<span class="sd"> compression_level : int, default None</span>
<span class="sd"> Use a compression level particular to the chosen compressor. If None</span>
<span class="sd"> use the default compression level</span>
<span class="sd"> chunksize : int, default None</span>
<span class="sd"> For V2 files, the internal maximum size of Arrow RecordBatch chunks</span>
<span class="sd"> when writing the Arrow IPC file format. None means use the default,</span>
<span class="sd"> which is currently 64K</span>
<span class="sd"> version : int, default 2</span>
<span class="sd"> Feather file version. Version 2 is the current. Version 1 is the more</span>
<span class="sd"> limited legacy format</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">_pandas_api</span><span class="o">.</span><span class="n">have_pandas</span><span class="p">:</span>
<span class="k">if</span> <span class="p">(</span><span class="n">_pandas_api</span><span class="o">.</span><span class="n">has_sparse</span> <span class="ow">and</span>
<span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">_pandas_api</span><span class="o">.</span><span class="n">pd</span><span class="o">.</span><span class="n">SparseDataFrame</span><span class="p">)):</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">to_dense</span><span class="p">()</span>
<span class="k">if</span> <span class="n">_pandas_api</span><span class="o">.</span><span class="n">is_data_frame</span><span class="p">(</span><span class="n">df</span><span class="p">):</span>
<span class="c1"># Feather v1 creates a new column in the resultant Table to</span>
<span class="c1"># store index information if index type is not RangeIndex</span>
<span class="k">if</span> <span class="n">version</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">preserve_index</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">elif</span> <span class="n">version</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
<span class="n">preserve_index</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Version value should either be 1 or 2&quot;</span><span class="p">)</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">Table</span><span class="o">.</span><span class="n">from_pandas</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">preserve_index</span><span class="o">=</span><span class="n">preserve_index</span><span class="p">)</span>
<span class="k">if</span> <span class="n">version</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="c1"># Version 1 does not chunking</span>
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">names</span><span class="p">):</span>
<span class="n">col</span> <span class="o">=</span> <span class="n">table</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
<span class="n">check_chunked_overflow</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">col</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">df</span>
<span class="k">if</span> <span class="n">version</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">column_names</span><span class="p">)</span> <span class="o">&gt;</span> <span class="nb">len</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">table</span><span class="o">.</span><span class="n">column_names</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;cannot serialize duplicate column names&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">compression</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Feather V1 files do not support compression &quot;</span>
<span class="s2">&quot;option&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">chunksize</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Feather V1 files do not support chunksize &quot;</span>
<span class="s2">&quot;option&quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compression</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">Codec</span><span class="o">.</span><span class="n">is_available</span><span class="p">(</span><span class="s1">&#39;lz4_frame&#39;</span><span class="p">):</span>
<span class="n">compression</span> <span class="o">=</span> <span class="s1">&#39;lz4&#39;</span>
<span class="k">elif</span> <span class="p">(</span><span class="n">compression</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
<span class="n">compression</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_FEATHER_SUPPORTED_CODECS</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;compression=&quot;</span><span class="si">{</span><span class="n">compression</span><span class="si">}</span><span class="s1">&quot; not supported, must be &#39;</span>
<span class="sa">f</span><span class="s1">&#39;one of </span><span class="si">{</span><span class="n">_FEATHER_SUPPORTED_CODECS</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">_feather</span><span class="o">.</span><span class="n">write_feather</span><span class="p">(</span><span class="n">table</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">compression</span><span class="o">=</span><span class="n">compression</span><span class="p">,</span>
<span class="n">compression_level</span><span class="o">=</span><span class="n">compression_level</span><span class="p">,</span>
<span class="n">chunksize</span><span class="o">=</span><span class="n">chunksize</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="n">version</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dest</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">dest</span><span class="p">)</span>
<span class="k">except</span> <span class="n">os</span><span class="o">.</span><span class="n">error</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">raise</span></div>
<div class="viewcode-block" id="read_feather">
<a class="viewcode-back" href="../../python/generated/pyarrow.feather.read_feather.html#pyarrow.feather.read_feather">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">read_feather</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">memory_map</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Read a pandas.DataFrame from Feather format. To read as pyarrow.Table use</span>
<span class="sd"> feather.read_table.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> source : str file path, or file-like object</span>
<span class="sd"> You can use MemoryMappedFile as source, for explicitly use memory map.</span>
<span class="sd"> columns : sequence, optional</span>
<span class="sd"> Only read a specific set of columns. If not provided, all columns are</span>
<span class="sd"> read.</span>
<span class="sd"> use_threads : bool, default True</span>
<span class="sd"> Whether to parallelize reading using multiple threads. If false the</span>
<span class="sd"> restriction is used in the conversion to Pandas as well as in the</span>
<span class="sd"> reading from Feather format.</span>
<span class="sd"> memory_map : boolean, default False</span>
<span class="sd"> Use memory mapping when opening file on disk, when source is a str.</span>
<span class="sd"> **kwargs</span>
<span class="sd"> Additional keyword arguments passed on to `pyarrow.Table.to_pandas`.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> df : pandas.DataFrame</span>
<span class="sd"> The contents of the Feather file as a pandas.DataFrame</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">(</span><span class="n">read_table</span><span class="p">(</span>
<span class="n">source</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">memory_map</span><span class="o">=</span><span class="n">memory_map</span><span class="p">,</span>
<span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span><span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">))</span></div>
<div class="viewcode-block" id="read_table">
<a class="viewcode-back" href="../../python/generated/pyarrow.feather.read_table.html#pyarrow.feather.read_table">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">read_table</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">memory_map</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Read a pyarrow.Table from Feather format</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> source : str file path, or file-like object</span>
<span class="sd"> You can use MemoryMappedFile as source, for explicitly use memory map.</span>
<span class="sd"> columns : sequence, optional</span>
<span class="sd"> Only read a specific set of columns. If not provided, all columns are</span>
<span class="sd"> read.</span>
<span class="sd"> memory_map : boolean, default False</span>
<span class="sd"> Use memory mapping when opening file on disk, when source is a str</span>
<span class="sd"> use_threads : bool, default True</span>
<span class="sd"> Whether to parallelize reading using multiple threads.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> table : pyarrow.Table</span>
<span class="sd"> The contents of the Feather file as a pyarrow.Table</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">reader</span> <span class="o">=</span> <span class="n">_feather</span><span class="o">.</span><span class="n">FeatherReader</span><span class="p">(</span>
<span class="n">source</span><span class="p">,</span> <span class="n">use_memory_map</span><span class="o">=</span><span class="n">memory_map</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span>
<span class="k">if</span> <span class="n">columns</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">reader</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">columns</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;Columns must be a sequence but, got </span><span class="si">{}</span><span class="s2">&quot;</span>
<span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
<span class="n">column_types</span> <span class="o">=</span> <span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="n">column</span><span class="p">)</span> <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">all</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span> <span class="o">==</span> <span class="nb">int</span><span class="p">,</span> <span class="n">column_types</span><span class="p">)):</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">reader</span><span class="o">.</span><span class="n">read_indices</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">all</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span> <span class="o">==</span> <span class="nb">str</span><span class="p">,</span> <span class="n">column_types</span><span class="p">)):</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">reader</span><span class="o">.</span><span class="n">read_names</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">column_type_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="vm">__name__</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">column_types</span><span class="p">]</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;Columns must be indices or names. &quot;</span>
<span class="sa">f</span><span class="s2">&quot;Got columns </span><span class="si">{</span><span class="n">columns</span><span class="si">}</span><span class="s2"> of types </span><span class="si">{</span><span class="n">column_type_names</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="c1"># Feather v1 already respects the column selection</span>
<span class="k">if</span> <span class="n">reader</span><span class="o">.</span><span class="n">version</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
<span class="k">return</span> <span class="n">table</span>
<span class="c1"># Feather v2 reads with sorted / deduplicated selection</span>
<span class="k">elif</span> <span class="nb">sorted</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">columns</span><span class="p">))</span> <span class="o">==</span> <span class="n">columns</span><span class="p">:</span>
<span class="k">return</span> <span class="n">table</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># follow exact order / selection of names</span>
<span class="k">return</span> <span class="n">table</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">columns</span><span class="p">)</span></div>
</pre></div>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
</div>
</footer>
</div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<p class="copyright">
© Copyright 2016-2025 Apache Software Foundation.
Apache Arrow, Arrow, Apache, the Apache logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
<br/>
</p>
</div>
<div class="footer-item">
<p class="sphinx-version">
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3.
<br/>
</p>
</div>
</div>
<div class="footer-items__end">
<div class="footer-item">
<p class="theme-version">
<!-- # L10n: Setting the PST URL as an argument as this does not need to be localized -->
Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.16.1.
</p></div>
</div>
</div>
</footer>
</body>
</html>