| |
| |
| <!DOCTYPE html> |
| |
| |
| <html lang="en" data-content_root="" > |
| |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| <title>pyarrow.fs — Apache Arrow v17.0.0.dev52</title> |
| |
| |
| |
| <script data-cfasync="false"> |
| document.documentElement.dataset.mode = localStorage.getItem("mode") || ""; |
| document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; |
| </script> |
| |
| <!-- Loaded before other Sphinx assets --> |
| <link href="../../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link href="../../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| |
| |
| <link href="../../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> |
| <link rel="preload" as="font" type="font/woff2" crossorigin href="../../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> |
| |
| <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/design-style.1e8bd061cd6da7fc9cf755528e8ffc24.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css" /> |
| |
| <!-- Pre-loaded scripts that we'll load fully later --> |
| <link rel="preload" as="script" href="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> |
| <link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> |
| <script src="../../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script> |
| <script src="../../_static/doctools.js"></script> |
| <script src="../../_static/sphinx_highlight.js"></script> |
| <script src="../../_static/clipboard.min.js"></script> |
| <script src="../../_static/copybutton.js"></script> |
| <script src="../../_static/design-tabs.js"></script> |
| <script>DOCUMENTATION_OPTIONS.pagename = '_modules/pyarrow/fs';</script> |
| <script> |
| DOCUMENTATION_OPTIONS.theme_version = '0.15.2'; |
| DOCUMENTATION_OPTIONS.theme_switcher_json_url = '/docs/_static/versions.json'; |
| DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'dev/'; |
| DOCUMENTATION_OPTIONS.show_version_warning_banner = true; |
| </script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/_modules/pyarrow/fs.html" /> |
| <link rel="icon" href="../../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../../genindex.html" /> |
| <link rel="search" title="Search" href="../../search.html" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1"/> |
| <meta name="docsearch:language" content="en"/> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| |
| |
| <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode=""> |
| |
| |
| |
| <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> |
| |
| <div id="pst-scroll-pixel-helper"></div> |
| |
| <button type="button" class="btn rounded-pill" id="pst-back-to-top"> |
| <i class="fa-solid fa-arrow-up"></i> |
| Back to top |
| </button> |
| |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__primary" |
| id="__primary"/> |
| <label class="overlay overlay-primary" for="__primary"></label> |
| |
| <input type="checkbox" |
| class="sidebar-toggle" |
| name="__secondary" |
| id="__secondary"/> |
| <label class="overlay overlay-secondary" for="__secondary"></label> |
| |
| <div class="search-button__wrapper"> |
| <div class="search-button__overlay"></div> |
| <div class="search-button__search-container"> |
| <form class="bd-search d-flex align-items-center" |
| action="../../search.html" |
| method="get"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <input type="search" |
| class="form-control" |
| name="q" |
| id="search-input" |
| placeholder="Search the docs ..." |
| aria-label="Search the docs ..." |
| autocomplete="off" |
| autocorrect="off" |
| autocapitalize="off" |
| spellcheck="false"/> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> |
| </form></div> |
| </div> |
| |
| <header class="bd-header navbar navbar-expand-lg bd-navbar"> |
| <div class="bd-header__inner bd-page-width"> |
| <label class="sidebar-toggle primary-toggle" for="__primary"> |
| <span class="fa-solid fa-bars"></span> |
| </label> |
| |
| |
| <div class="col-lg-3 navbar-header-items__start"> |
| |
| <div class="navbar-item"> |
| |
| |
| |
| <a class="navbar-brand logo" href="../../index.html"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <img src="../../_static/arrow.png" class="logo__image only-light" alt="Apache Arrow v17.0.0.dev52 - Home"/> |
| <script>document.write(`<img src="../../_static/arrow-dark.png" class="logo__image only-dark" alt="Apache Arrow v17.0.0.dev52 - Home"/>`);</script> |
| |
| |
| </a></div> |
| |
| </div> |
| |
| <div class="col-lg-9 navbar-header-items"> |
| |
| <div class="me-auto navbar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| <div class="navbar-header-items__end"> |
| |
| <div class="navbar-item navbar-persistent--container"> |
| |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-2" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-2" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-2" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-2"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="navbar-persistent--mobile"> |
| |
| <script> |
| document.write(` |
| <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <i class="fa-solid fa-magnifying-glass"></i> |
| <span class="search-button__default-text">Search</span> |
| <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> |
| </button> |
| `); |
| </script> |
| </div> |
| |
| |
| |
| </div> |
| |
| </header> |
| |
| |
| <div class="bd-container"> |
| <div class="bd-container__inner bd-page-width"> |
| |
| |
| |
| |
| |
| <div class="bd-sidebar-primary bd-sidebar hide-on-wide"> |
| |
| |
| |
| <div class="sidebar-header-items sidebar-primary__section"> |
| |
| |
| <div class="sidebar-header-items__center"> |
| |
| <div class="navbar-item"> |
| <nav class="navbar-nav"> |
| <ul class="bd-navbar-elements navbar-nav"> |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../format/index.html"> |
| Specifications |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link nav-internal" href="../../developers/index.html"> |
| Development |
| </a> |
| </li> |
| |
| <li class="nav-item dropdown"> |
| <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links-2"> |
| Implementations |
| </button> |
| <ul id="pst-nav-more-links-2" class="dropdown-menu"> |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../cpp/index.html"> |
| C++ |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://pkg.go.dev/github.com/apache/arrow/go/v17"> |
| Go |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../java/index.html"> |
| Java |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/julia/"> |
| Julia |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/nanoarrow/"> |
| nanoarrow |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../python/index.html"> |
| Python |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../r/index.html"> |
| R |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://github.com/apache/arrow/blob/main/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-internal" href="../../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/java/"> |
| Java cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/py/"> |
| Python cookbook |
| </a> |
| </li> |
| |
| |
| <li class="nav-item"> |
| <a class="nav-link dropdown-item nav-external" href="https://arrow.apache.org/cookbook/r/"> |
| R cookbook |
| </a> |
| </li> |
| |
| </ul> |
| </li> |
| |
| </ul> |
| </nav></div> |
| |
| </div> |
| |
| |
| |
| <div class="sidebar-header-items__end"> |
| |
| <div class="navbar-item"> |
| <script> |
| document.write(` |
| <div class="version-switcher__container dropdown"> |
| <button id="pst-version-switcher-button-3" |
| type="button" |
| class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle" |
| data-bs-toggle="dropdown" |
| aria-haspopup="listbox" |
| aria-controls="pst-version-switcher-list-3" |
| aria-label="Version switcher list" |
| > |
| Choose version <!-- this text may get changed later by javascript --> |
| <span class="caret"></span> |
| </button> |
| <div id="pst-version-switcher-list-3" |
| class="version-switcher__menu dropdown-menu list-group-flush py-0" |
| role="listbox" aria-labelledby="pst-version-switcher-button-3"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"> |
| |
| <script> |
| document.write(` |
| <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> |
| <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> |
| <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> |
| </button> |
| `); |
| </script></div> |
| |
| <div class="navbar-item"><ul class="navbar-icon-links navbar-nav" |
| aria-label="Icon Links"> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://github.com/apache/arrow" title="GitHub" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-github fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">GitHub</span></a> |
| </li> |
| <li class="nav-item"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <a href="https://twitter.com/ApacheArrow" title="X" class="nav-link" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><span><i class="fa-brands fa-square-x-twitter fa-lg" aria-hidden="true"></i></span> |
| <span class="sr-only">X</span></a> |
| </li> |
| </ul></div> |
| |
| </div> |
| |
| </div> |
| |
| |
| <div class="sidebar-primary-items__end sidebar-primary__section"> |
| </div> |
| |
| <div id="rtd-footer-container"></div> |
| |
| |
| </div> |
| |
| <main id="main-content" class="bd-main"> |
| |
| |
| <div class="bd-content"> |
| <div class="bd-article-container"> |
| |
| <div class="bd-header-article"> |
| <div class="header-article-items header-article__inner"> |
| |
| <div class="header-article-items__start"> |
| |
| <div class="header-article-item"> |
| |
| |
| |
| <nav aria-label="Breadcrumb"> |
| <ul class="bd-breadcrumbs"> |
| |
| <li class="breadcrumb-item breadcrumb-home"> |
| <a href="../../index.html" class="nav-link" aria-label="Home"> |
| <i class="fa-solid fa-home"></i> |
| </a> |
| </li> |
| |
| <li class="breadcrumb-item"><a href="../index.html" class="nav-link">Module code</a></li> |
| |
| |
| <li class="breadcrumb-item"><a href="../pyarrow.html" class="nav-link">pyarrow</a></li> |
| |
| <li class="breadcrumb-item active" aria-current="page">pyarrow.fs</li> |
| </ul> |
| </nav> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| |
| |
| |
| |
| <div id="searchbox"></div> |
| <article class="bd-article"> |
| |
| <h1>Source code for pyarrow.fs</h1><div class="highlight"><pre> |
| <span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="c1"># or more contributor license agreements. See the NOTICE file</span> |
| <span class="c1"># distributed with this work for additional information</span> |
| <span class="c1"># regarding copyright ownership. The ASF licenses this file</span> |
| <span class="c1"># to you under the Apache License, Version 2.0 (the</span> |
| <span class="c1"># "License"); you may not use this file except in compliance</span> |
| <span class="c1"># with the License. You may obtain a copy of the License at</span> |
| <span class="c1">#</span> |
| <span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="c1">#</span> |
| <span class="c1"># Unless required by applicable law or agreed to in writing,</span> |
| <span class="c1"># software distributed under the License is distributed on an</span> |
| <span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> |
| <span class="c1"># KIND, either express or implied. See the License for the</span> |
| <span class="c1"># specific language governing permissions and limitations</span> |
| <span class="c1"># under the License.</span> |
| |
| <span class="sd">"""</span> |
| <span class="sd">FileSystem abstraction to interact with various local and remote filesystems.</span> |
| <span class="sd">"""</span> |
| |
| <span class="kn">from</span> <span class="nn">pyarrow.util</span> <span class="kn">import</span> <span class="n">_is_path_like</span><span class="p">,</span> <span class="n">_stringify_path</span> |
| |
| <span class="kn">from</span> <span class="nn">pyarrow._fs</span> <span class="kn">import</span> <span class="p">(</span> <span class="c1"># noqa</span> |
| <span class="n">FileSelector</span><span class="p">,</span> |
| <span class="n">FileType</span><span class="p">,</span> |
| <span class="n">FileInfo</span><span class="p">,</span> |
| <span class="n">FileSystem</span><span class="p">,</span> |
| <span class="n">LocalFileSystem</span><span class="p">,</span> |
| <span class="n">SubTreeFileSystem</span><span class="p">,</span> |
| <span class="n">_MockFileSystem</span><span class="p">,</span> |
| <span class="n">FileSystemHandler</span><span class="p">,</span> |
| <span class="n">PyFileSystem</span><span class="p">,</span> |
| <span class="n">_copy_files</span><span class="p">,</span> |
| <span class="n">_copy_files_selector</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="c1"># For backward compatibility.</span> |
| <span class="n">FileStats</span> <span class="o">=</span> <span class="n">FileInfo</span> |
| |
| <span class="n">_not_imported</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">pyarrow._azurefs</span> <span class="kn">import</span> <span class="n">AzureFileSystem</span> <span class="c1"># noqa</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="n">_not_imported</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"AzureFileSystem"</span><span class="p">)</span> |
| |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">pyarrow._hdfs</span> <span class="kn">import</span> <span class="n">HadoopFileSystem</span> <span class="c1"># noqa</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="n">_not_imported</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"HadoopFileSystem"</span><span class="p">)</span> |
| |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">pyarrow._gcsfs</span> <span class="kn">import</span> <span class="n">GcsFileSystem</span> <span class="c1"># noqa</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="n">_not_imported</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"GcsFileSystem"</span><span class="p">)</span> |
| |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">pyarrow._s3fs</span> <span class="kn">import</span> <span class="p">(</span> <span class="c1"># noqa</span> |
| <span class="n">AwsDefaultS3RetryStrategy</span><span class="p">,</span> <span class="n">AwsStandardS3RetryStrategy</span><span class="p">,</span> |
| <span class="n">S3FileSystem</span><span class="p">,</span> <span class="n">S3LogLevel</span><span class="p">,</span> <span class="n">S3RetryStrategy</span><span class="p">,</span> <span class="n">ensure_s3_initialized</span><span class="p">,</span> |
| <span class="n">finalize_s3</span><span class="p">,</span> <span class="n">ensure_s3_finalized</span><span class="p">,</span> <span class="n">initialize_s3</span><span class="p">,</span> <span class="n">resolve_s3_region</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="n">_not_imported</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"S3FileSystem"</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="c1"># GH-38364: we don't initialize S3 eagerly as that could lead</span> |
| <span class="c1"># to crashes at shutdown even when S3 isn't used.</span> |
| <span class="c1"># Instead, S3 is initialized lazily using `ensure_s3_initialized`</span> |
| <span class="c1"># in assorted places.</span> |
| <span class="kn">import</span> <span class="nn">atexit</span> |
| <span class="n">atexit</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">ensure_s3_finalized</span><span class="p">)</span> |
| |
| |
| <span class="k">def</span> <span class="fm">__getattr__</span><span class="p">(</span><span class="n">name</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">_not_imported</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ImportError</span><span class="p">(</span> |
| <span class="s2">"The pyarrow installation is not built with support for "</span> |
| <span class="s2">"'</span><span class="si">{0}</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> |
| <span class="p">)</span> |
| |
| <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span> |
| <span class="s2">"module 'pyarrow.fs' has no attribute '</span><span class="si">{0}</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> |
| <span class="p">)</span> |
| |
| |
| <span class="k">def</span> <span class="nf">_filesystem_from_str</span><span class="p">(</span><span class="n">uri</span><span class="p">):</span> |
| <span class="c1"># instantiate the file system from an uri, if the uri has a path</span> |
| <span class="c1"># component then it will be treated as a path prefix</span> |
| <span class="n">filesystem</span><span class="p">,</span> <span class="n">prefix</span> <span class="o">=</span> <span class="n">FileSystem</span><span class="o">.</span><span class="n">from_uri</span><span class="p">(</span><span class="n">uri</span><span class="p">)</span> |
| <span class="n">prefix</span> <span class="o">=</span> <span class="n">filesystem</span><span class="o">.</span><span class="n">normalize_path</span><span class="p">(</span><span class="n">prefix</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">prefix</span><span class="p">:</span> |
| <span class="c1"># validate that the prefix is pointing to a directory</span> |
| <span class="n">prefix_info</span> <span class="o">=</span> <span class="n">filesystem</span><span class="o">.</span><span class="n">get_file_info</span><span class="p">([</span><span class="n">prefix</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span> |
| <span class="k">if</span> <span class="n">prefix_info</span><span class="o">.</span><span class="n">type</span> <span class="o">!=</span> <span class="n">FileType</span><span class="o">.</span><span class="n">Directory</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <span class="s2">"The path component of the filesystem URI must point to a "</span> |
| <span class="s2">"directory but it has a type: `</span><span class="si">{}</span><span class="s2">`. The path component "</span> |
| <span class="s2">"is `</span><span class="si">{}</span><span class="s2">` and the given filesystem URI is `</span><span class="si">{}</span><span class="s2">`"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> |
| <span class="n">prefix_info</span><span class="o">.</span><span class="n">type</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">prefix_info</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">uri</span> |
| <span class="p">)</span> |
| <span class="p">)</span> |
| <span class="n">filesystem</span> <span class="o">=</span> <span class="n">SubTreeFileSystem</span><span class="p">(</span><span class="n">prefix</span><span class="p">,</span> <span class="n">filesystem</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">filesystem</span> |
| |
| |
| <span class="k">def</span> <span class="nf">_ensure_filesystem</span><span class="p">(</span><span class="n">filesystem</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">use_mmap</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">filesystem</span><span class="p">,</span> <span class="n">FileSystem</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">filesystem</span> |
| <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">filesystem</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">use_mmap</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <span class="s2">"Specifying to use memory mapping not supported for "</span> |
| <span class="s2">"filesystem specified as an URI string"</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="n">_filesystem_from_str</span><span class="p">(</span><span class="n">filesystem</span><span class="p">)</span> |
| |
| <span class="c1"># handle fsspec-compatible filesystems</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="kn">import</span> <span class="nn">fsspec</span> |
| <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span> |
| <span class="k">pass</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">filesystem</span><span class="p">,</span> <span class="n">fsspec</span><span class="o">.</span><span class="n">AbstractFileSystem</span><span class="p">):</span> |
| <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">filesystem</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'LocalFileSystem'</span><span class="p">:</span> |
| <span class="c1"># In case its a simple LocalFileSystem, use native arrow one</span> |
| <span class="k">return</span> <span class="n">LocalFileSystem</span><span class="p">(</span><span class="n">use_mmap</span><span class="o">=</span><span class="n">use_mmap</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">PyFileSystem</span><span class="p">(</span><span class="n">FSSpecHandler</span><span class="p">(</span><span class="n">filesystem</span><span class="p">))</span> |
| |
| <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span> |
| <span class="s2">"Unrecognized filesystem: </span><span class="si">{}</span><span class="s2">. `filesystem` argument must be a "</span> |
| <span class="s2">"FileSystem instance or a valid file system URI'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> |
| <span class="nb">type</span><span class="p">(</span><span class="n">filesystem</span><span class="p">))</span> |
| <span class="p">)</span> |
| |
| |
| <span class="k">def</span> <span class="nf">_resolve_filesystem_and_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">filesystem</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">memory_map</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Return filesystem/path from path which could be an URI or a plain</span> |
| <span class="sd"> filesystem path.</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">_is_path_like</span><span class="p">(</span><span class="n">path</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">filesystem</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <span class="s2">"'filesystem' passed but the specified path is file-like, so"</span> |
| <span class="s2">" there is nothing to open with 'filesystem'."</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="n">filesystem</span><span class="p">,</span> <span class="n">path</span> |
| |
| <span class="k">if</span> <span class="n">filesystem</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">filesystem</span> <span class="o">=</span> <span class="n">_ensure_filesystem</span><span class="p">(</span><span class="n">filesystem</span><span class="p">,</span> <span class="n">use_mmap</span><span class="o">=</span><span class="n">memory_map</span><span class="p">)</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">filesystem</span><span class="p">,</span> <span class="n">LocalFileSystem</span><span class="p">):</span> |
| <span class="n">path</span> <span class="o">=</span> <span class="n">_stringify_path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span> |
| <span class="s2">"Expected string path; path-like objects are only allowed "</span> |
| <span class="s2">"with a local filesystem"</span> |
| <span class="p">)</span> |
| <span class="n">path</span> <span class="o">=</span> <span class="n">filesystem</span><span class="o">.</span><span class="n">normalize_path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">filesystem</span><span class="p">,</span> <span class="n">path</span> |
| |
| <span class="n">path</span> <span class="o">=</span> <span class="n">_stringify_path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| |
| <span class="c1"># if filesystem is not given, try to automatically determine one</span> |
| <span class="c1"># first check if the file exists as a local (relative) file path</span> |
| <span class="c1"># if not then try to parse the path as an URI</span> |
| <span class="n">filesystem</span> <span class="o">=</span> <span class="n">LocalFileSystem</span><span class="p">(</span><span class="n">use_mmap</span><span class="o">=</span><span class="n">memory_map</span><span class="p">)</span> |
| |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">file_info</span> <span class="o">=</span> <span class="n">filesystem</span><span class="o">.</span><span class="n">get_file_info</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> <span class="c1"># ValueError means path is likely an URI</span> |
| <span class="n">file_info</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="n">exists_locally</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">exists_locally</span> <span class="o">=</span> <span class="p">(</span><span class="n">file_info</span><span class="o">.</span><span class="n">type</span> <span class="o">!=</span> <span class="n">FileType</span><span class="o">.</span><span class="n">NotFound</span><span class="p">)</span> |
| |
| <span class="c1"># if the file or directory doesn't exists locally, then assume that</span> |
| <span class="c1"># the path is an URI describing the file system as well</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">exists_locally</span><span class="p">:</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">filesystem</span><span class="p">,</span> <span class="n">path</span> <span class="o">=</span> <span class="n">FileSystem</span><span class="o">.</span><span class="n">from_uri</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <span class="c1"># neither an URI nor a locally existing path, so assume that</span> |
| <span class="c1"># local path was given and propagate a nicer file not found error</span> |
| <span class="c1"># instead of a more confusing scheme parsing error</span> |
| <span class="k">if</span> <span class="s2">"empty scheme"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> \ |
| <span class="ow">and</span> <span class="s2">"Cannot parse URI"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">):</span> |
| <span class="k">raise</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">path</span> <span class="o">=</span> <span class="n">filesystem</span><span class="o">.</span><span class="n">normalize_path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">filesystem</span><span class="p">,</span> <span class="n">path</span> |
| |
| |
| <div class="viewcode-block" id="copy_files"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.copy_files.html#pyarrow.fs.copy_files">[docs]</a><span class="k">def</span> <span class="nf">copy_files</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">destination</span><span class="p">,</span> |
| <span class="n">source_filesystem</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">destination_filesystem</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="o">*</span><span class="p">,</span> <span class="n">chunk_size</span><span class="o">=</span><span class="mi">1024</span><span class="o">*</span><span class="mi">1024</span><span class="p">,</span> <span class="n">use_threads</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Copy files between FileSystems.</span> |
| |
| <span class="sd"> This functions allows you to recursively copy directories of files from</span> |
| <span class="sd"> one file system to another, such as from S3 to your local machine.</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> source : string</span> |
| <span class="sd"> Source file path or URI to a single file or directory.</span> |
| <span class="sd"> If a directory, files will be copied recursively from this path.</span> |
| <span class="sd"> destination : string</span> |
| <span class="sd"> Destination file path or URI. If `source` is a file, `destination`</span> |
| <span class="sd"> is also interpreted as the destination file (not directory).</span> |
| <span class="sd"> Directories will be created as necessary.</span> |
| <span class="sd"> source_filesystem : FileSystem, optional</span> |
| <span class="sd"> Source filesystem, needs to be specified if `source` is not a URI,</span> |
| <span class="sd"> otherwise inferred.</span> |
| <span class="sd"> destination_filesystem : FileSystem, optional</span> |
| <span class="sd"> Destination filesystem, needs to be specified if `destination` is not</span> |
| <span class="sd"> a URI, otherwise inferred.</span> |
| <span class="sd"> chunk_size : int, default 1MB</span> |
| <span class="sd"> The maximum size of block to read before flushing to the</span> |
| <span class="sd"> destination file. A larger chunk_size will use more memory while</span> |
| <span class="sd"> copying but may help accommodate high latency FileSystems.</span> |
| <span class="sd"> use_threads : bool, default True</span> |
| <span class="sd"> Whether to use multiple threads to accelerate copying.</span> |
| |
| <span class="sd"> Examples</span> |
| <span class="sd"> --------</span> |
| <span class="sd"> Inspect an S3 bucket's files:</span> |
| |
| <span class="sd"> >>> s3, path = fs.FileSystem.from_uri(</span> |
| <span class="sd"> ... "s3://registry.opendata.aws/roda/ndjson/")</span> |
| <span class="sd"> >>> selector = fs.FileSelector(path)</span> |
| <span class="sd"> >>> s3.get_file_info(selector)</span> |
| <span class="sd"> [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]</span> |
| |
| <span class="sd"> Copy one file from S3 bucket to a local directory:</span> |
| |
| <span class="sd"> >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",</span> |
| <span class="sd"> ... "file:///{}/index_copy.ndjson".format(local_path))</span> |
| |
| <span class="sd"> >>> fs.LocalFileSystem().get_file_info(str(local_path)+</span> |
| <span class="sd"> ... '/index_copy.ndjson')</span> |
| <span class="sd"> <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...></span> |
| |
| <span class="sd"> Copy file using a FileSystem object:</span> |
| |
| <span class="sd"> >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",</span> |
| <span class="sd"> ... "file:///{}/index_copy.ndjson".format(local_path),</span> |
| <span class="sd"> ... source_filesystem=fs.S3FileSystem())</span> |
| <span class="sd"> """</span> |
| <span class="n">source_fs</span><span class="p">,</span> <span class="n">source_path</span> <span class="o">=</span> <span class="n">_resolve_filesystem_and_path</span><span class="p">(</span> |
| <span class="n">source</span><span class="p">,</span> <span class="n">source_filesystem</span> |
| <span class="p">)</span> |
| <span class="n">destination_fs</span><span class="p">,</span> <span class="n">destination_path</span> <span class="o">=</span> <span class="n">_resolve_filesystem_and_path</span><span class="p">(</span> |
| <span class="n">destination</span><span class="p">,</span> <span class="n">destination_filesystem</span> |
| <span class="p">)</span> |
| |
| <span class="n">file_info</span> <span class="o">=</span> <span class="n">source_fs</span><span class="o">.</span><span class="n">get_file_info</span><span class="p">(</span><span class="n">source_path</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">file_info</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="n">FileType</span><span class="o">.</span><span class="n">Directory</span><span class="p">:</span> |
| <span class="n">source_sel</span> <span class="o">=</span> <span class="n">FileSelector</span><span class="p">(</span><span class="n">source_path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| <span class="n">_copy_files_selector</span><span class="p">(</span><span class="n">source_fs</span><span class="p">,</span> <span class="n">source_sel</span><span class="p">,</span> |
| <span class="n">destination_fs</span><span class="p">,</span> <span class="n">destination_path</span><span class="p">,</span> |
| <span class="n">chunk_size</span><span class="p">,</span> <span class="n">use_threads</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">_copy_files</span><span class="p">(</span><span class="n">source_fs</span><span class="p">,</span> <span class="n">source_path</span><span class="p">,</span> |
| <span class="n">destination_fs</span><span class="p">,</span> <span class="n">destination_path</span><span class="p">,</span> |
| <span class="n">chunk_size</span><span class="p">,</span> <span class="n">use_threads</span><span class="p">)</span></div> |
| |
| |
| <div class="viewcode-block" id="FSSpecHandler"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler">[docs]</a><span class="k">class</span> <span class="nc">FSSpecHandler</span><span class="p">(</span><span class="n">FileSystemHandler</span><span class="p">):</span> |
| <span class="w"> </span><span class="sd">"""</span> |
| <span class="sd"> Handler for fsspec-based Python filesystems.</span> |
| |
| <span class="sd"> https://filesystem-spec.readthedocs.io/en/latest/index.html</span> |
| |
| <span class="sd"> Parameters</span> |
| <span class="sd"> ----------</span> |
| <span class="sd"> fs : FSSpec-compliant filesystem instance</span> |
| |
| <span class="sd"> Examples</span> |
| <span class="sd"> --------</span> |
| <span class="sd"> >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP</span> |
| <span class="sd"> """</span> |
| |
| <div class="viewcode-block" id="FSSpecHandler.__init__"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.__init__">[docs]</a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fs</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span> <span class="o">=</span> <span class="n">fs</span></div> |
| |
| <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">FSSpecHandler</span><span class="p">):</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">fs</span> |
| <span class="k">return</span> <span class="bp">NotImplemented</span> |
| |
| <span class="k">def</span> <span class="fm">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">FSSpecHandler</span><span class="p">):</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span> <span class="o">!=</span> <span class="n">other</span><span class="o">.</span><span class="n">fs</span> |
| <span class="k">return</span> <span class="bp">NotImplemented</span> |
| |
| <div class="viewcode-block" id="FSSpecHandler.get_type_name"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.get_type_name">[docs]</a> <span class="k">def</span> <span class="nf">get_type_name</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="n">protocol</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">protocol</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">protocol</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span> |
| <span class="n">protocol</span> <span class="o">=</span> <span class="n">protocol</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> |
| <span class="k">return</span> <span class="s2">"fsspec+</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">protocol</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.normalize_path"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.normalize_path">[docs]</a> <span class="k">def</span> <span class="nf">normalize_path</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">path</span></div> |
| |
| <span class="nd">@staticmethod</span> |
| <span class="k">def</span> <span class="nf">_create_file_info</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">info</span><span class="p">):</span> |
| <span class="n">size</span> <span class="o">=</span> <span class="n">info</span><span class="p">[</span><span class="s2">"size"</span><span class="p">]</span> |
| <span class="k">if</span> <span class="n">info</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"file"</span><span class="p">:</span> |
| <span class="n">ftype</span> <span class="o">=</span> <span class="n">FileType</span><span class="o">.</span><span class="n">File</span> |
| <span class="k">elif</span> <span class="n">info</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"directory"</span><span class="p">:</span> |
| <span class="n">ftype</span> <span class="o">=</span> <span class="n">FileType</span><span class="o">.</span><span class="n">Directory</span> |
| <span class="c1"># some fsspec filesystems include a file size for directories</span> |
| <span class="n">size</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">ftype</span> <span class="o">=</span> <span class="n">FileType</span><span class="o">.</span><span class="n">Unknown</span> |
| <span class="k">return</span> <span class="n">FileInfo</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">ftype</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="n">size</span><span class="p">,</span> <span class="n">mtime</span><span class="o">=</span><span class="n">info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"mtime"</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span> |
| |
| <div class="viewcode-block" id="FSSpecHandler.get_file_info"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.get_file_info">[docs]</a> <span class="k">def</span> <span class="nf">get_file_info</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">paths</span><span class="p">):</span> |
| <span class="n">infos</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">for</span> <span class="n">path</span> <span class="ow">in</span> <span class="n">paths</span><span class="p">:</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span> |
| <span class="n">infos</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">FileInfo</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">FileType</span><span class="o">.</span><span class="n">NotFound</span><span class="p">))</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">infos</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_create_file_info</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">info</span><span class="p">))</span> |
| <span class="k">return</span> <span class="n">infos</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.get_file_info_selector"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.get_file_info_selector">[docs]</a> <span class="k">def</span> <span class="nf">get_file_info_selector</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">selector</span><span class="p">):</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">selector</span><span class="o">.</span><span class="n">base_dir</span><span class="p">):</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">selector</span><span class="o">.</span><span class="n">base_dir</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">NotADirectoryError</span><span class="p">(</span><span class="n">selector</span><span class="o">.</span><span class="n">base_dir</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">selector</span><span class="o">.</span><span class="n">allow_not_found</span><span class="p">:</span> |
| <span class="k">return</span> <span class="p">[]</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="n">selector</span><span class="o">.</span><span class="n">base_dir</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">selector</span><span class="o">.</span><span class="n">recursive</span><span class="p">:</span> |
| <span class="n">maxdepth</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">maxdepth</span> <span class="o">=</span> <span class="mi">1</span> |
| |
| <span class="n">infos</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="n">selected_files</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">find</span><span class="p">(</span> |
| <span class="n">selector</span><span class="o">.</span><span class="n">base_dir</span><span class="p">,</span> <span class="n">maxdepth</span><span class="o">=</span><span class="n">maxdepth</span><span class="p">,</span> <span class="n">withdirs</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">detail</span><span class="o">=</span><span class="kc">True</span> |
| <span class="p">)</span> |
| <span class="k">for</span> <span class="n">path</span><span class="p">,</span> <span class="n">info</span> <span class="ow">in</span> <span class="n">selected_files</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> |
| <span class="n">_path</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span> |
| <span class="n">base_dir</span> <span class="o">=</span> <span class="n">selector</span><span class="o">.</span><span class="n">base_dir</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span> |
| <span class="c1"># Need to exclude base directory from selected files if present</span> |
| <span class="c1"># (fsspec filesystems, see GH-37555)</span> |
| <span class="k">if</span> <span class="n">_path</span> <span class="o">!=</span> <span class="n">base_dir</span><span class="p">:</span> |
| <span class="n">infos</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_create_file_info</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">info</span><span class="p">))</span> |
| |
| <span class="k">return</span> <span class="n">infos</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.create_dir"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.create_dir">[docs]</a> <span class="k">def</span> <span class="nf">create_dir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="p">):</span> |
| <span class="c1"># mkdir also raises FileNotFoundError when base directory is not found</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">create_parents</span><span class="o">=</span><span class="n">recursive</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">FileExistsError</span><span class="p">:</span> |
| <span class="k">pass</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.delete_dir"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.delete_dir">[docs]</a> <span class="k">def</span> <span class="nf">delete_dir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">rm</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> |
| |
| <span class="k">def</span> <span class="nf">_delete_dir_contents</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">missing_dir_ok</span><span class="p">):</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">subpaths</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">detail</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">missing_dir_ok</span><span class="p">:</span> |
| <span class="k">return</span> |
| <span class="k">raise</span> |
| <span class="k">for</span> <span class="n">subpath</span> <span class="ow">in</span> <span class="n">subpaths</span><span class="p">:</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">subpath</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">rm</span><span class="p">(</span><span class="n">subpath</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">subpath</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">rm</span><span class="p">(</span><span class="n">subpath</span><span class="p">)</span> |
| |
| <div class="viewcode-block" id="FSSpecHandler.delete_dir_contents"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.delete_dir_contents">[docs]</a> <span class="k">def</span> <span class="nf">delete_dir_contents</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">missing_dir_ok</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| <span class="s2">"delete_dir_contents called on path '"</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_delete_dir_contents</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">missing_dir_ok</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.delete_root_dir_contents"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.delete_root_dir_contents">[docs]</a> <span class="k">def</span> <span class="nf">delete_root_dir_contents</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_delete_dir_contents</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.delete_file"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.delete_file">[docs]</a> <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> |
| <span class="c1"># fs.rm correctly raises IsADirectoryError when `path` is a directory</span> |
| <span class="c1"># instead of a file and `recursive` is not set to True</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">path</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">rm</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.move"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.move">[docs]</a> <span class="k">def</span> <span class="nf">move</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">mv</span><span class="p">(</span><span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.copy_file"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.copy_file">[docs]</a> <span class="k">def</span> <span class="nf">copy_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">):</span> |
| <span class="c1"># fs.copy correctly raises IsADirectoryError when `src` is a directory</span> |
| <span class="c1"># instead of a file</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">)</span></div> |
| |
| <span class="c1"># TODO can we read/pass metadata (e.g. Content-Type) in the methods below?</span> |
| |
| <div class="viewcode-block" id="FSSpecHandler.open_input_stream"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.open_input_stream">[docs]</a> <span class="k">def</span> <span class="nf">open_input_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> |
| <span class="kn">from</span> <span class="nn">pyarrow</span> <span class="kn">import</span> <span class="n">PythonFile</span> |
| |
| <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">path</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">PythonFile</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"rb"</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"r"</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.open_input_file"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.open_input_file">[docs]</a> <span class="k">def</span> <span class="nf">open_input_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> |
| <span class="kn">from</span> <span class="nn">pyarrow</span> <span class="kn">import</span> <span class="n">PythonFile</span> |
| |
| <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">path</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">PythonFile</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"rb"</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"r"</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.open_output_stream"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.open_output_stream">[docs]</a> <span class="k">def</span> <span class="nf">open_output_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">metadata</span><span class="p">):</span> |
| <span class="kn">from</span> <span class="nn">pyarrow</span> <span class="kn">import</span> <span class="n">PythonFile</span> |
| |
| <span class="k">return</span> <span class="n">PythonFile</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"wb"</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"w"</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="FSSpecHandler.open_append_stream"><a class="viewcode-back" href="../../python/generated/pyarrow.fs.FSSpecHandler.html#pyarrow.fs.FSSpecHandler.open_append_stream">[docs]</a> <span class="k">def</span> <span class="nf">open_append_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">metadata</span><span class="p">):</span> |
| <span class="kn">from</span> <span class="nn">pyarrow</span> <span class="kn">import</span> <span class="n">PythonFile</span> |
| |
| <span class="k">return</span> <span class="n">PythonFile</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"ab"</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"w"</span><span class="p">)</span></div></div> |
| </pre></div> |
| |
| </article> |
| |
| |
| |
| |
| |
| <footer class="prev-next-footer"> |
| |
| <div class="prev-next-area"> |
| </div> |
| </footer> |
| |
| </div> |
| |
| |
| |
| <div class="bd-sidebar-secondary bd-toc"></div> |
| |
| |
| </div> |
| <footer class="bd-footer-content"> |
| |
| </footer> |
| |
| </main> |
| </div> |
| </div> |
| |
| <!-- Scripts loaded after <body> so the DOM is not blocked --> |
| <script src="../../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script> |
| <script src="../../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script> |
| |
| <footer class="bd-footer"> |
| <div class="bd-footer__inner bd-page-width"> |
| |
| <div class="footer-items__start"> |
| |
| <div class="footer-item"> |
| |
| <p class="copyright"> |
| |
| © Copyright 2016-2024 Apache Software Foundation. |
| Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. |
| <br/> |
| |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| |
| <p class="sphinx-version"> |
| Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 6.2.0. |
| <br/> |
| </p> |
| </div> |
| |
| </div> |
| |
| |
| |
| <div class="footer-items__end"> |
| |
| <div class="footer-item"> |
| <p class="theme-version"> |
| Built with the <a href="https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html">PyData Sphinx Theme</a> 0.15.2. |
| </p></div> |
| |
| </div> |
| |
| </div> |
| |
| </footer> |
| </body> |
| </html> |
