| |
| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| <title>pyarrow.ipc — Apache Arrow v17.0.0.dev52</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; |
| </script> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| |
| |
| <link href="../../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> |
| |
| <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> |
| <link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> |
| <script src="../../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script> |
| <script src="../../_static/doctools.js"></script> |
| <script src="../../_static/sphinx_highlight.js"></script> |
| <script src="../../_static/clipboard.min.js"></script> |
| <script src="../../_static/copybutton.js"></script> |
| <script src="../../_static/design-tabs.js"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = '_modules/pyarrow/ipc';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.15.2'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/_modules/pyarrow/ipc.html" /> |
| <link rel="icon" href="../../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../../genindex.html" /> |
| <link rel="search" title="Search" href="../../search.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i> |
| Back to top |
| </button> |
| |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__primary" |
| id="__primary"/> |
| <label class="overlay overlay-primary" for="__primary"></label> |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__secondary" |
| id="__secondary"/> |
| <label class="overlay overlay-secondary" for="__secondary"></label> |
| |
| <div class="search-button__wrapper"> |
| <div class="search-button__overlay"></div> |
| <div class="search-button__search-container"> |
| <form class="bd-search d-flex align-items-center" |
| action="../../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| id="search-input" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form></div> |
| </div> |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar"> |
| <div class="bd-header__inner bd-page-width"> |
| <label class="sidebar-toggle primary-toggle" for="__primary"> |
| <span class="fa-solid fa-bars"></span> |
| </label> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| <a class="navbar-brand logo" href="../../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev52 - Home"/> |
| <script>document.write(`<img src="../../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev52 - Home"/>`);</script> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| |
| |
| <div class="bd-sidebar-primary bd-sidebar hide-on-wide"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links-2" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| </div> |
| |
| <div id="rtd-footer-container"></div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| |
| |
| <nav aria-label="Breadcrumb"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Module code</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="../pyarrow.html" class="nav-link">pyarrow</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page">pyarrow.ipc</li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <h1>Source code for pyarrow.ipc</h1><div class="highlight"><pre> |
| <span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="c1"># or more contributor license agreements. See the NOTICE file</span> |
| <span class="c1"># distributed with this work for additional information</span> |
| <span class="c1"># regarding copyright ownership. The ASF licenses this file</span> |
| <span class="c1"># to you under the Apache License, Version 2.0 (the</span> |
| <span class="c1"># "License"); you may not use this file except in compliance</span> |
| <span class="c1"># with the License. You may obtain a copy of the License at</span> |
| <span class="c1">#</span> |
| <span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="c1">#</span> |
| <span class="c1"># Unless required by applicable law or agreed to in writing,</span> |
| <span class="c1"># software distributed under the License is distributed on an</span> |
| <span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> |
| <span class="c1"># KIND, either express or implied. See the License for the</span> |
| <span class="c1"># specific language governing permissions and limitations</span> |
| <span class="c1"># under the License.</span> |
| |
| <span class="c1"># Arrow file and stream reader/writer classes, and other messaging tools</span> |
| |
| <span class="kn">import</span> <span class="nn">os</span> |
| |
| <span class="kn">import</span> <span class="nn">pyarrow</span> <span class="k">as</span> <span class="nn">pa</span> |
| |
| <span class="kn">from</span> <span class="nn">pyarrow.lib</span> <span class="kn">import</span> <span class="p">(</span><span class="n">IpcReadOptions</span><span class="p">,</span> <span class="n">IpcWriteOptions</span><span class="p">,</span> <span class="n">ReadStats</span><span class="p">,</span> <span class="n">WriteStats</span><span class="p">,</span> <span class="c1"># noqa</span> |
| <span class="n">Message</span><span class="p">,</span> <span class="n">MessageReader</span><span class="p">,</span> |
| <span class="n">RecordBatchReader</span><span class="p">,</span> <span class="n">_ReadPandasMixin</span><span class="p">,</span> |
| <span class="n">MetadataVersion</span><span class="p">,</span> |
| <span class="n">read_message</span><span class="p">,</span> <span class="n">read_record_batch</span><span class="p">,</span> <span class="n">read_schema</span><span class="p">,</span> |
| <span class="n">read_tensor</span><span class="p">,</span> <span class="n">write_tensor</span><span class="p">,</span> |
| <span class="n">get_record_batch_size</span><span class="p">,</span> <span class="n">get_tensor_size</span><span class="p">)</span> |
| <span class="kn">import</span> <span class="nn">pyarrow.lib</span> <span class="k">as</span> <span class="nn">lib</span> |
| |
| |
| <div class="viewcode-block" id="RecordBatchStreamReader"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamReader.html#pyarrow.ipc.RecordBatchStreamReader">[docs]</a><span class="k">class</span> <span class="nc">RecordBatchStreamReader</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchStreamReader</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Reader for the Arrow streaming binary format.</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span> |
| <span class="sd"> Either an in-memory buffer, or a readable file object.</span> |
| <span class="sd"> If you want to use memory map use MemoryMappedFile as source.</span> |
| <span class="sd"> options : pyarrow.ipc.IpcReadOptions</span> |
| <span class="sd"> Options for IPC deserialization.</span> |
| <span class="sd"> If None, default values will be used.</span> |
| <span class="sd"> memory_pool : MemoryPool, default None</span> |
| <span class="sd"> If None, default memory pool is used.</span> |
| <span class="sd"> """</span> |
| |
| <div class="viewcode-block" id="RecordBatchStreamReader.__init__"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamReader.html#pyarrow.ipc.RecordBatchStreamReader.__init__">[docs]</a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="n">options</span> <span class="o">=</span> <span class="n">_ensure_default_ipc_read_options</span><span class="p">(</span><span class="n">options</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div></div> |
| |
| |
| <span class="n">_ipc_writer_class_doc</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span> |
| <span class="s2">Parameters</span> |
| <span class="s2">----------</span> |
| <span class="s2">sink : str, pyarrow.NativeFile, or file-like Python object</span> |
| <span class="s2"> Either a file path, or a writable file object.</span> |
| <span class="s2">schema : pyarrow.Schema</span> |
| <span class="s2"> The Arrow schema for data to be written to the file.</span> |
| <span class="s2">use_legacy_format : bool, default None</span> |
| <span class="s2"> Deprecated in favor of setting options. Cannot be provided with</span> |
| <span class="s2"> options.</span> |
| |
| <span class="s2"> If None, False will be used unless this default is overridden by</span> |
| <span class="s2"> setting the environment variable ARROW_PRE_0_15_IPC_FORMAT=1</span> |
| <span class="s2">options : pyarrow.ipc.IpcWriteOptions</span> |
| <span class="s2"> Options for IPC serialization.</span> |
| |
| <span class="s2"> If None, default values will be used: the legacy format will not</span> |
| <span class="s2"> be used unless overridden by setting the environment variable</span> |
| <span class="s2"> ARROW_PRE_0_15_IPC_FORMAT=1, and the V5 metadata version will be</span> |
| <span class="s2"> used unless overridden by setting the environment variable</span> |
| <span class="s2"> ARROW_PRE_1_0_METADATA_VERSION=1."""</span> |
| |
| |
| <div class="viewcode-block" id="RecordBatchStreamWriter"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamWriter.html#pyarrow.ipc.RecordBatchStreamWriter">[docs]</a><span class="k">class</span> <span class="nc">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchStreamWriter</span><span class="p">):</span> |
| <span class="vm">__doc__</span> <span class="o">=</span> <span class="s2">"""Writer for the Arrow streaming binary format</span> |
| |
| <span class="si">{}</span><span class="s2">"""</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">_ipc_writer_class_doc</span><span class="p">)</span> |
| |
| <div class="viewcode-block" id="RecordBatchStreamWriter.__init__"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchStreamWriter.html#pyarrow.ipc.RecordBatchStreamWriter.__init__">[docs]</a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">use_legacy_format</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="n">options</span> <span class="o">=</span> <span class="n">_get_legacy_format_default</span><span class="p">(</span><span class="n">use_legacy_format</span><span class="p">,</span> <span class="n">options</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span></div></div> |
| |
| |
| <div class="viewcode-block" id="RecordBatchFileReader"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileReader.html#pyarrow.ipc.RecordBatchFileReader">[docs]</a><span class="k">class</span> <span class="nc">RecordBatchFileReader</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchFileReader</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Class for reading Arrow record batch data from the Arrow binary file format</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span> |
| <span class="sd"> Either an in-memory buffer, or a readable file object.</span> |
| <span class="sd"> If you want to use memory map use MemoryMappedFile as source.</span> |
| <span class="sd"> footer_offset : int, default None</span> |
| <span class="sd"> If the file is embedded in some larger file, this is the byte offset to</span> |
| <span class="sd"> the very end of the file data</span> |
| <span class="sd"> options : pyarrow.ipc.IpcReadOptions</span> |
| <span class="sd"> Options for IPC serialization.</span> |
| <span class="sd"> If None, default values will be used.</span> |
| <span class="sd"> memory_pool : MemoryPool, default None</span> |
| <span class="sd"> If None, default memory pool is used.</span> |
| <span class="sd"> """</span> |
| |
| <div class="viewcode-block" id="RecordBatchFileReader.__init__"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileReader.html#pyarrow.ipc.RecordBatchFileReader.__init__">[docs]</a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="n">options</span> <span class="o">=</span> <span class="n">_ensure_default_ipc_read_options</span><span class="p">(</span><span class="n">options</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="n">footer_offset</span><span class="p">,</span> |
| <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div></div> |
| |
| |
| <div class="viewcode-block" id="RecordBatchFileWriter"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileWriter.html#pyarrow.ipc.RecordBatchFileWriter">[docs]</a><span class="k">class</span> <span class="nc">RecordBatchFileWriter</span><span class="p">(</span><span class="n">lib</span><span class="o">.</span><span class="n">_RecordBatchFileWriter</span><span class="p">):</span> |
| |
| <span class="vm">__doc__</span> <span class="o">=</span> <span class="s2">"""Writer to create the Arrow binary file format</span> |
| |
| <span class="si">{}</span><span class="s2">"""</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">_ipc_writer_class_doc</span><span class="p">)</span> |
| |
| <div class="viewcode-block" id="RecordBatchFileWriter.__init__"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.RecordBatchFileWriter.html#pyarrow.ipc.RecordBatchFileWriter.__init__">[docs]</a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">use_legacy_format</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="n">options</span> <span class="o">=</span> <span class="n">_get_legacy_format_default</span><span class="p">(</span><span class="n">use_legacy_format</span><span class="p">,</span> <span class="n">options</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_open</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span></div></div> |
| |
| |
| <span class="k">def</span> <span class="nf">_get_legacy_format_default</span><span class="p">(</span><span class="n">use_legacy_format</span><span class="p">,</span> <span class="n">options</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">use_legacy_format</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">options</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <span class="s2">"Can provide at most one of options and use_legacy_format"</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="n">options</span><span class="p">:</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">options</span><span class="p">,</span> <span class="n">IpcWriteOptions</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"expected IpcWriteOptions, got </span><span class="si">{}</span><span class="s2">"</span> |
| <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">options</span><span class="p">)))</span> |
| <span class="k">return</span> <span class="n">options</span> |
| |
| <span class="n">metadata_version</span> <span class="o">=</span> <span class="n">MetadataVersion</span><span class="o">.</span><span class="n">V5</span> |
| <span class="k">if</span> <span class="n">use_legacy_format</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">use_legacy_format</span> <span class="o">=</span> \ |
| <span class="nb">bool</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ARROW_PRE_0_15_IPC_FORMAT'</span><span class="p">,</span> <span class="s1">'0'</span><span class="p">)))</span> |
| <span class="k">if</span> <span class="nb">bool</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ARROW_PRE_1_0_METADATA_VERSION'</span><span class="p">,</span> <span class="s1">'0'</span><span class="p">))):</span> |
| <span class="n">metadata_version</span> <span class="o">=</span> <span class="n">MetadataVersion</span><span class="o">.</span><span class="n">V4</span> |
| <span class="k">return</span> <span class="n">IpcWriteOptions</span><span class="p">(</span><span class="n">use_legacy_format</span><span class="o">=</span><span class="n">use_legacy_format</span><span class="p">,</span> |
| <span class="n">metadata_version</span><span class="o">=</span><span class="n">metadata_version</span><span class="p">)</span> |
| |
| |
| <span class="k">def</span> <span class="nf">_ensure_default_ipc_read_options</span><span class="p">(</span><span class="n">options</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">options</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">options</span><span class="p">,</span> <span class="n">IpcReadOptions</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span> |
| <span class="s2">"expected IpcReadOptions, got </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">options</span><span class="p">))</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="n">options</span> <span class="ow">or</span> <span class="n">IpcReadOptions</span><span class="p">()</span> |
| |
| |
| <div class="viewcode-block" id="new_stream"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.new_stream.html#pyarrow.ipc.new_stream">[docs]</a><span class="k">def</span> <span class="nf">new_stream</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">use_legacy_format</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> |
| <span class="n">use_legacy_format</span><span class="o">=</span><span class="n">use_legacy_format</span><span class="p">,</span> |
| <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span></div> |
| |
| |
| <span class="n">new_stream</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span> |
| <span class="s2">Create an Arrow columnar IPC stream writer instance</span> |
| |
| <span class="si">{}</span> |
| |
| <span class="s2">Returns</span> |
| <span class="s2">-------</span> |
| <span class="s2">writer : RecordBatchStreamWriter</span> |
| <span class="s2"> A writer for the given sink</span> |
| <span class="s2">"""</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">_ipc_writer_class_doc</span><span class="p">)</span> |
| |
| |
| <div class="viewcode-block" id="open_stream"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.open_stream.html#pyarrow.ipc.open_stream">[docs]</a><span class="k">def</span> <span class="nf">open_stream</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Create reader for Arrow streaming format.</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span> |
| <span class="sd"> Either an in-memory buffer, or a readable file object.</span> |
| <span class="sd"> options : pyarrow.ipc.IpcReadOptions</span> |
| <span class="sd"> Options for IPC serialization.</span> |
| <span class="sd"> If None, default values will be used.</span> |
| <span class="sd"> memory_pool : MemoryPool, default None</span> |
| <span class="sd"> If None, default memory pool is used.</span> |
| |
| <span class="sd"> Returns</span> |
| <span class="sd"> -------</span> |
| <span class="sd"> reader : RecordBatchStreamReader</span> |
| <span class="sd"> A reader for the given source</span> |
| <span class="sd"> """</span> |
| <span class="k">return</span> <span class="n">RecordBatchStreamReader</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> |
| <span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div> |
| |
| |
| <div class="viewcode-block" id="new_file"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.new_file.html#pyarrow.ipc.new_file">[docs]</a><span class="k">def</span> <span class="nf">new_file</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">use_legacy_format</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">RecordBatchFileWriter</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> |
| <span class="n">use_legacy_format</span><span class="o">=</span><span class="n">use_legacy_format</span><span class="p">,</span> |
| <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">)</span></div> |
| |
| |
| <span class="n">new_file</span><span class="o">.</span><span class="vm">__doc__</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span> |
| <span class="s2">Create an Arrow columnar IPC file writer instance</span> |
| |
| <span class="si">{}</span> |
| |
| <span class="s2">Returns</span> |
| <span class="s2">-------</span> |
| <span class="s2">writer : RecordBatchFileWriter</span> |
| <span class="s2"> A writer for the given sink</span> |
| <span class="s2">"""</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">_ipc_writer_class_doc</span><span class="p">)</span> |
| |
| |
| <div class="viewcode-block" id="open_file"><a class="viewcode-back" href="../../python/generated/pyarrow.ipc.open_file.html#pyarrow.ipc.open_file">[docs]</a><span class="k">def</span> <span class="nf">open_file</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Create reader for Arrow file format.</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> source : bytes/buffer-like, pyarrow.NativeFile, or file-like Python object</span> |
| <span class="sd"> Either an in-memory buffer, or a readable file object.</span> |
| <span class="sd"> footer_offset : int, default None</span> |
| <span class="sd"> If the file is embedded in some larger file, this is the byte offset to</span> |
| <span class="sd"> the very end of the file data.</span> |
| <span class="sd"> options : pyarrow.ipc.IpcReadOptions</span> |
| <span class="sd"> Options for IPC serialization.</span> |
| <span class="sd"> If None, default values will be used.</span> |
| <span class="sd"> memory_pool : MemoryPool, default None</span> |
| <span class="sd"> If None, default memory pool is used.</span> |
| |
| <span class="sd"> Returns</span> |
| <span class="sd"> -------</span> |
| <span class="sd"> reader : RecordBatchFileReader</span> |
| <span class="sd"> A reader for the given source</span> |
| <span class="sd"> """</span> |
| <span class="k">return</span> <span class="n">RecordBatchFileReader</span><span class="p">(</span> |
| <span class="n">source</span><span class="p">,</span> <span class="n">footer_offset</span><span class="o">=</span><span class="n">footer_offset</span><span class="p">,</span> |
| <span class="n">options</span><span class="o">=</span><span class="n">options</span><span class="p">,</span> <span class="n">memory_pool</span><span class="o">=</span><span class="n">memory_pool</span><span class="p">)</span></div> |
| |
| |
| <span class="k">def</span> <span class="nf">serialize_pandas</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">nthreads</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">preserve_index</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Serialize a pandas DataFrame into a buffer protocol compatible object.</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> df : pandas.DataFrame</span> |
| <span class="sd"> nthreads : int, default None</span> |
| <span class="sd"> Number of threads to use for conversion to Arrow, default all CPUs.</span> |
| <span class="sd"> preserve_index : bool, default None</span> |
| <span class="sd"> The default of None will store the index as a column, except for</span> |
| <span class="sd"> RangeIndex which is stored as metadata only. If True, always</span> |
| <span class="sd"> preserve the pandas index data as a column. If False, no index</span> |
| <span class="sd"> information is saved and the result will have a default RangeIndex.</span> |
| |
| <span class="sd"> Returns</span> |
| <span class="sd"> -------</span> |
| <span class="sd"> buf : buffer</span> |
| <span class="sd"> An object compatible with the buffer protocol.</span> |
| <span class="sd"> """</span> |
| <span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_pandas</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span> |
| <span class="n">preserve_index</span><span class="o">=</span><span class="n">preserve_index</span><span class="p">)</span> |
| <span class="n">sink</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">BufferOutputStream</span><span class="p">()</span> |
| <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchStreamWriter</span><span class="p">(</span><span class="n">sink</span><span class="p">,</span> <span class="n">batch</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span> |
| <span class="n">writer</span><span class="o">.</span><span class="n">write_batch</span><span class="p">(</span><span class="n">batch</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">sink</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span> |
| |
| |
| <span class="k">def</span> <span class="nf">deserialize_pandas</span><span class="p">(</span><span class="n">buf</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""Deserialize a buffer protocol compatible object into a pandas DataFrame.</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> buf : buffer</span> |
| <span class="sd"> An object compatible with the buffer protocol.</span> |
| <span class="sd"> use_threads : bool, default True</span> |
| <span class="sd"> Whether to parallelize the conversion using multiple threads.</span> |
| |
| <span class="sd"> Returns</span> |
| <span class="sd"> -------</span> |
| <span class="sd"> df : pandas.DataFrame</span> |
| <span class="sd"> The buffer deserialized as pandas DataFrame</span> |
| <span class="sd"> """</span> |
| <span class="n">buffer_reader</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">BufferReader</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span> |
| <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatchStreamReader</span><span class="p">(</span><span class="n">buffer_reader</span><span class="p">)</span> <span class="k">as</span> <span class="n">reader</span><span class="p">:</span> |
| <span class="n">table</span> <span class="o">=</span> <span class="n">reader</span><span class="o">.</span><span class="n">read_all</span><span class="p">()</span> |
| <span class="k">return</span> <span class="n">table</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">(</span><span class="n">use_threads</span><span class="o">=</span><span class="n">use_threads</span><span class="p">)</span> |
| </pre></div> |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer"> |
| |
| <div class="prev-next-area"> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <div class="bd-sidebar-secondary bd-toc"></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script> |
| <script src="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2024 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
