| |
| <!DOCTYPE html> |
| |
| <html> |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| <title>pyarrow.fs.HadoopFileSystem — Apache Arrow v7.0.0</title> |
| |
| <link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> |
| <link href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet"> |
| |
| |
| <link rel="stylesheet" |
| href="../../_static/vendor/fontawesome/5.13.0/css/all.min.css"> |
| <link rel="preload" as="font" type="font/woff2" crossorigin |
| href="../../_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2"> |
| <link rel="preload" as="font" type="font/woff2" crossorigin |
| href="../../_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2"> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" type="text/css" href="../../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/styles/pydata-sphinx-theme.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/tabs.css" /> |
| <link rel="stylesheet" type="text/css" href="../../_static/theme_overrides.css" /> |
| |
| <link rel="preload" as="script" href="../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"> |
| |
| <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script> |
| <script src="../../_static/jquery.js"></script> |
| <script src="../../_static/underscore.js"></script> |
| <script src="../../_static/doctools.js"></script> |
| <link rel="canonical" href="https://arrow.apache.org/docs/python/generated/pyarrow.fs.HadoopFileSystem.html" /> |
| <link rel="shortcut icon" href="../../_static/favicon.ico"/> |
| <link rel="index" title="Index" href="../../genindex.html" /> |
| <link rel="search" title="Search" href="../../search.html" /> |
| <link rel="next" title="pyarrow.fs.SubTreeFileSystem" href="pyarrow.fs.SubTreeFileSystem.html" /> |
| <link rel="prev" title="pyarrow.fs.S3FileSystem" href="pyarrow.fs.S3FileSystem.html" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1" /> |
| <meta name="docsearch:language" content="None"> |
| |
| |
| <!-- Google Analytics --> |
| |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '20']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| </head> |
| <body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80"> |
| |
| <div class="container-fluid" id="banner"></div> |
| |
| |
| |
| |
| <div class="container-xl"> |
| <div class="row"> |
| |
| |
| <!-- Only show if we have sidebars configured, else just a small margin --> |
| <div class="col-12 col-md-3 bd-sidebar"> |
| <div class="sidebar-start-items"> |
| <a class="navbar-brand" href="../../index.html"> |
| <img src="../../_static/arrow.png" class="logo" alt="logo"> |
| </a> |
| |
| <div id="version-search-wrapper"> |
| |
| <div id="version-button" class="dropdown"> |
| <button type="button" class="btn btn-secondary btn-sm navbar-btn dropdown-toggle" id="version_switcher_button" data-toggle="dropdown"> |
| 7.0.0 |
| <span class="caret"></span> |
| </button> |
| <div id="version_switcher" class="dropdown-menu list-group-flush py-0" aria-labelledby="version_switcher_button"> |
| <!-- dropdown will be populated by javascript on page load --> |
| </div> |
| </div> |
| |
| <script type="text/javascript"> |
| // Function to construct the target URL from the JSON components |
| function buildURL(entry) { |
| var template = "https://arrow.apache.org/docs/{version}"; // supplied by jinja |
| template = template.replace("{version}", entry.version); |
| return template; |
| } |
| |
| // Function to check if corresponding page path exists in other version of docs |
| // and, if so, go there instead of the homepage of the other docs version |
| function checkPageExistsAndRedirect(event) { |
| const currentFilePath = "python/generated/pyarrow.fs.HadoopFileSystem.html", |
| otherDocsHomepage = event.target.getAttribute("href"); |
| let tryUrl = `${otherDocsHomepage}${currentFilePath}`; |
| $.ajax({ |
| type: 'HEAD', |
| url: tryUrl, |
| // if the page exists, go there |
| success: function() { |
| location.href = tryUrl; |
| } |
| }).fail(function() { |
| location.href = otherDocsHomepage; |
| }); |
| return false; |
| } |
| |
| // Function to populate the version switcher |
| (function () { |
| // get JSON config |
| $.getJSON("/docs/_static/versions.json", function(data, textStatus, jqXHR) { |
| // create the nodes first (before AJAX calls) to ensure the order is |
| // correct (for now, links will go to doc version homepage) |
| $.each(data, function(index, entry) { |
| // if no custom name specified (e.g., "latest"), use version string |
| if (!("name" in entry)) { |
| entry.name = entry.version; |
| } |
| // construct the appropriate URL, and add it to the dropdown |
| entry.url = buildURL(entry); |
| const node = document.createElement("a"); |
| node.setAttribute("class", "list-group-item list-group-item-action py-1"); |
| node.setAttribute("href", `${entry.url}`); |
| node.textContent = `${entry.name}`; |
| node.onclick = checkPageExistsAndRedirect; |
| $("#version_switcher").append(node); |
| }); |
| }); |
| })(); |
| </script> |
| |
| <form id="search-box" class="bd-search d-flex align-items-center" action="../../search.html" method="get"> |
| <i class="icon fas fa-search"></i> |
| <input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" > |
| </form> |
| |
| </div> |
| |
| <nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation"> |
| <div class="bd-toc-item active"> |
| |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Supported Environments |
| </span> |
| </p> |
| <ul class="current nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../c_glib/index.html"> |
| C/GLib |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../../cpp/index.html"> |
| C++ |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" type="checkbox"/> |
| <label for="toctree-checkbox-1"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../../cpp/getting_started.html"> |
| User Guide |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" type="checkbox"/> |
| <label for="toctree-checkbox-2"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/overview.html"> |
| High-Level Overview |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/conventions.html"> |
| Conventions |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/build_system.html"> |
| Using Arrow C++ in your own project |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/memory.html"> |
| Memory Management |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/arrays.html"> |
| Arrays |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/datatypes.html"> |
| Data Types |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/tables.html"> |
| Tabular Data |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/compute.html"> |
| Compute Functions |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/streaming_execution.html"> |
| Streaming execution engine |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/io.html"> |
| Input / output and filesystems |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/ipc.html"> |
| Reading and writing the Arrow IPC format |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/parquet.html"> |
| Reading and writing Parquet files |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/csv.html"> |
| Reading and Writing CSV files |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/json.html"> |
| Reading JSON files |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/dataset.html"> |
| Tabular Datasets |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/gdb.html"> |
| Debugging code using Arrow |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../../cpp/examples/index.html"> |
| Examples |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" type="checkbox"/> |
| <label for="toctree-checkbox-3"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/examples/cmake_minimal_build.html"> |
| Minimal build using CMake |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/examples/compute_and_write_example.html"> |
| Compute and Write CSV Example |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/examples/dataset_documentation_example.html"> |
| Arrow Datasets example |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/examples/row_columnar_conversion.html"> |
| Row to columnar conversion |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/examples/tuple_range_conversion.html"> |
| std::tuple-like ranges to Arrow |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../../cpp/api.html"> |
| API Reference |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" type="checkbox"/> |
| <label for="toctree-checkbox-4"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/support.html"> |
| Programming Support |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/memory.html"> |
| Memory (management) |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/datatype.html"> |
| Data Types |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/array.html"> |
| Arrays |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/scalar.html"> |
| Scalars |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/builder.html"> |
| Array Builders |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/table.html"> |
| Two-dimensional Datasets |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/c_abi.html"> |
| C Interfaces |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/compute.html"> |
| Compute Functions |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/tensor.html"> |
| Tensors |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/utilities.html"> |
| Utilities |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/async.html"> |
| Asynchronous programming |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/io.html"> |
| Input / output |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/ipc.html"> |
| Arrow IPC |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/formats.html"> |
| File Formats |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/cuda.html"> |
| CUDA support |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/filesystem.html"> |
| Filesystems |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../cpp/api/dataset.html"> |
| Dataset |
| </a> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/csharp/README.md"> |
| C# |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://pkg.go.dev/github.com/apache/arrow/go"> |
| Go |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../../java/index.html"> |
| Java |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" type="checkbox"/> |
| <label for="toctree-checkbox-5"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../java/vector.html"> |
| ValueVector |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../java/vector_schema_root.html"> |
| VectorSchemaRoot |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../java/ipc.html"> |
| Reading/Writing IPC formats |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../java/algorithm.html"> |
| Java Algorithms |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../java/dataset.html"> |
| Dataset |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../java/reference/index.html"> |
| Reference (javadoc) |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../js/index.html"> |
| JavaScript |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/julia/Arrow/README.md"> |
| Julia |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/matlab/README.md"> |
| MATLAB |
| </a> |
| </li> |
| <li class="toctree-l1 current active has-children"> |
| <a class="reference internal" href="../index.html"> |
| Python |
| </a> |
| <input checked="" class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" type="checkbox"/> |
| <label for="toctree-checkbox-6"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul class="current"> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../install.html"> |
| Installing PyArrow |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../getstarted.html"> |
| Getting Started |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../data.html"> |
| Data Types and In-Memory Data Model |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../compute.html"> |
| Compute Functions |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../memory.html"> |
| Memory and IO Interfaces |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../ipc.html"> |
| Streaming, Serialization, and IPC |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../filesystems.html"> |
| Filesystem Interface |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../filesystems_deprecated.html"> |
| Filesystem Interface (legacy) |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" type="checkbox"/> |
| <label for="toctree-checkbox-7"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.hdfs.connect.html"> |
| pyarrow.hdfs.connect |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.cat.html"> |
| pyarrow.HadoopFileSystem.cat |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.chmod.html"> |
| pyarrow.HadoopFileSystem.chmod |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.chown.html"> |
| pyarrow.HadoopFileSystem.chown |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.delete.html"> |
| pyarrow.HadoopFileSystem.delete |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.df.html"> |
| pyarrow.HadoopFileSystem.df |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.disk_usage.html"> |
| pyarrow.HadoopFileSystem.disk_usage |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.download.html"> |
| pyarrow.HadoopFileSystem.download |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.exists.html"> |
| pyarrow.HadoopFileSystem.exists |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.get_capacity.html"> |
| pyarrow.HadoopFileSystem.get_capacity |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.get_space_used.html"> |
| pyarrow.HadoopFileSystem.get_space_used |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.info.html"> |
| pyarrow.HadoopFileSystem.info |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.ls.html"> |
| pyarrow.HadoopFileSystem.ls |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.mkdir.html"> |
| pyarrow.HadoopFileSystem.mkdir |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.open.html"> |
| pyarrow.HadoopFileSystem.open |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.rename.html"> |
| pyarrow.HadoopFileSystem.rename |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.rm.html"> |
| pyarrow.HadoopFileSystem.rm |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HadoopFileSystem.upload.html"> |
| pyarrow.HadoopFileSystem.upload |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="pyarrow.HdfsFile.html"> |
| pyarrow.HdfsFile |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../plasma.html"> |
| The Plasma In-Memory Object Store |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../numpy.html"> |
| NumPy Integration |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../pandas.html"> |
| Pandas Integration |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../timestamps.html"> |
| Timestamps |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../csv.html"> |
| Reading and Writing CSV files |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../feather.html"> |
| Feather File Format |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../json.html"> |
| Reading JSON files |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../parquet.html"> |
| Reading and Writing the Apache Parquet Format |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../dataset.html"> |
| Tabular Datasets |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../extending_types.html"> |
| Extending pyarrow |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../integration.html"> |
| PyArrow Integrations |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" type="checkbox"/> |
| <label for="toctree-checkbox-8"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../integration/python_r.html"> |
| Integrating PyArrow with R |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../integration/extending.html"> |
| Using pyarrow from C++ and Cython Code |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../integration/cuda.html"> |
| CUDA Integration |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2 current active has-children"> |
| <a class="reference internal" href="../api.html"> |
| API Reference |
| </a> |
| <input checked="" class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" type="checkbox"/> |
| <label for="toctree-checkbox-9"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul class="current"> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/datatypes.html"> |
| Data Types and Schemas |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/arrays.html"> |
| Arrays and Scalars |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/memory.html"> |
| Buffers and Memory |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/compute.html"> |
| Compute Functions |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/files.html"> |
| Streams and File Access |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/tables.html"> |
| Tables and Tensors |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/ipc.html"> |
| Serialization and IPC |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/flight.html"> |
| Arrow Flight |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/formats.html"> |
| Tabular File Formats |
| </a> |
| </li> |
| <li class="toctree-l3 current active"> |
| <a class="reference internal" href="../api/filesystems.html"> |
| Filesystems |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/dataset.html"> |
| Dataset |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/plasma.html"> |
| Plasma In-Memory Object Store |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/cuda.html"> |
| CUDA Integration |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../api/misc.html"> |
| Miscellaneous |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../getting_involved.html"> |
| Getting Involved |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../benchmarks.html"> |
| Benchmarks |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../r/index.html"> |
| R |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://github.com/apache/arrow/blob/master/ruby/README.md"> |
| Ruby |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://docs.rs/crate/arrow/"> |
| Rust |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../status.html"> |
| Implementation Status |
| </a> |
| </li> |
| </ul> |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Cookbooks |
| </span> |
| </p> |
| <ul class="nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://arrow.apache.org/cookbook/cpp/"> |
| C++ |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://arrow.apache.org/cookbook/py/"> |
| Python |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference external" href="https://arrow.apache.org/cookbook/r/"> |
| R |
| </a> |
| </li> |
| </ul> |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Specifications and Protocols |
| </span> |
| </p> |
| <ul class="nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../format/Versioning.html"> |
| Format Versioning and Stability |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../format/Columnar.html"> |
| Arrow Columnar Format |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../format/Flight.html"> |
| Arrow Flight RPC |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../format/Integration.html"> |
| Integration Testing |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../format/CDataInterface.html"> |
| The Arrow C data interface |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../format/CStreamInterface.html"> |
| The Arrow C stream interface |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../format/Other.html"> |
| Other Data Structures |
| </a> |
| </li> |
| </ul> |
| <p aria-level="2" class="caption" role="heading"> |
| <span class="caption-text"> |
| Development |
| </span> |
| </p> |
| <ul class="nav bd-sidenav"> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../developers/contributing.html"> |
| Contributing to Apache Arrow |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../../developers/guide/index.html"> |
| New Contributor’s Guide |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" type="checkbox"/> |
| <label for="toctree-checkbox-10"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/guide/architectural_overview.html"> |
| Architectural Overview |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/guide/communication.html"> |
| Communication |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../../developers/guide/step_by_step/index.html"> |
| Steps in making your first PR |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" type="checkbox"/> |
| <label for="toctree-checkbox-11"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/step_by_step/set_up.html"> |
| Set up |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/step_by_step/building.html"> |
| Building the Arrow libraries 🏋🏿♀️ |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/step_by_step/finding_issues.html"> |
| Finding good first issues 🔎 |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/step_by_step/arrow_codebase.html"> |
| Working on the Arrow codebase 🧐 |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/step_by_step/testing.html"> |
| Testing 🧪 |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/step_by_step/pr_and_github.html"> |
| Lifecycle of a pull request |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/guide/documentation.html"> |
| Helping with documentation |
| </a> |
| </li> |
| <li class="toctree-l2 has-children"> |
| <a class="reference internal" href="../../developers/guide/tutorials/index.html"> |
| Tutorials |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" type="checkbox"/> |
| <label for="toctree-checkbox-12"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/tutorials/python_tutorial.html"> |
| Python tutorial |
| </a> |
| </li> |
| <li class="toctree-l3"> |
| <a class="reference internal" href="../../developers/guide/tutorials/r_tutorial.html"> |
| R tutorial |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/guide/resources.html"> |
| Additional information and resources |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../developers/reviewing.html"> |
| Reviewing contributions |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../../developers/cpp/index.html"> |
| C++ Development |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-13" name="toctree-checkbox-13" type="checkbox"/> |
| <label for="toctree-checkbox-13"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/cpp/building.html"> |
| Building Arrow C++ |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/cpp/development.html"> |
| Development Guidelines |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/cpp/windows.html"> |
| Developing on Windows |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/cpp/conventions.html"> |
| Conventions |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/cpp/fuzzing.html"> |
| Fuzzing Arrow C++ |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../developers/python.html"> |
| Python Development |
| </a> |
| </li> |
| <li class="toctree-l1 has-children"> |
| <a class="reference internal" href="../../developers/continuous_integration/index.html"> |
| Continuous Integration |
| </a> |
| <input class="toctree-checkbox" id="toctree-checkbox-14" name="toctree-checkbox-14" type="checkbox"/> |
| <label for="toctree-checkbox-14"> |
| <i class="fas fa-chevron-down"> |
| </i> |
| </label> |
| <ul> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/continuous_integration/overview.html"> |
| Continuous Integration |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/continuous_integration/docker.html"> |
| Running Docker Builds |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/continuous_integration/archery.html"> |
| Daily Development using Archery |
| </a> |
| </li> |
| <li class="toctree-l2"> |
| <a class="reference internal" href="../../developers/continuous_integration/crossbow.html"> |
| Packaging and Testing with Crossbow |
| </a> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../developers/benchmarks.html"> |
| Benchmarks |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../developers/documentation.html"> |
| Building the Documentation |
| </a> |
| </li> |
| <li class="toctree-l1"> |
| <a class="reference internal" href="../../developers/computeir.html"> |
| Arrow Compute IR (Intermediate Representation) |
| </a> |
| </li> |
| </ul> |
| |
| |
| </div> |
| </nav> |
| </div> |
| <div class="sidebar-end-items"> |
| </div> |
| </div> |
| |
| |
| |
| |
| <div class="d-none d-xl-block col-xl-2 bd-toc"> |
| |
| |
| <div class="toc-item"> |
| |
| |
| <nav id="bd-toc-nav"> |
| |
| </nav> |
| </div> |
| |
| <div class="toc-item"> |
| |
| |
| <div class="tocsection editthispage"> |
| <a href="https://github.com/apache/arrow/edit/master/docs/source/python/generated/pyarrow.fs.HadoopFileSystem.rst"> |
| <i class="fas fa-pencil-alt"></i> Edit this page |
| </a> |
| </div> |
| |
| </div> |
| |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| <main class="col-12 col-md-9 col-xl-7 py-md-5 pl-md-5 pr-md-4 bd-content" role="main"> |
| |
| <div> |
| |
| <div class="section" id="pyarrow-fs-hadoopfilesystem"> |
| <h1>pyarrow.fs.HadoopFileSystem<a class="headerlink" href="#pyarrow-fs-hadoopfilesystem" title="Permalink to this headline">¶</a></h1> |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">pyarrow.fs.</span></span><span class="sig-name descname"><span class="pre">HadoopFileSystem</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">unicode</span> <span class="pre">host</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">int</span> <span class="pre">port=8020</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">unicode</span> <span class="pre">user=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">int</span> <span class="pre">replication=3</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">int</span> <span class="pre">buffer_size=0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">default_block_size=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">kerb_ticket=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">extra_conf=None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Bases: <a class="reference internal" href="pyarrow.fs.FileSystem.html#pyarrow.fs.FileSystem" title="pyarrow._fs.FileSystem"><code class="xref py py-class docutils literal notranslate"><span class="pre">pyarrow._fs.FileSystem</span></code></a></p> |
| <p>HDFS backed FileSystem implementation</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>host</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>HDFS host to connect to. Set to “default” for fs.defaultFS from |
| core-site.xml.</p> |
| </dd> |
| <dt><strong>port</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">int</span></code></a>, default 8020</span></dt><dd><p>HDFS port to connect to. Set to 0 for default or logical (HA) nodes.</p> |
| </dd> |
| <dt><strong>user</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a>, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>Username when connecting to HDFS; None implies login user.</p> |
| </dd> |
| <dt><strong>replication</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">int</span></code></a>, default 3</span></dt><dd><p>Number of copies each block will have.</p> |
| </dd> |
| <dt><strong>buffer_size</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">int</span></code></a>, default 0</span></dt><dd><p>If 0, no buffering will happen otherwise the size of the temporary read |
| and write buffer.</p> |
| </dd> |
| <dt><strong>default_block_size</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">int</span></code></a>, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>None means the default configuration for HDFS, a typical block size is |
| 128 MB.</p> |
| </dd> |
| <dt><strong>kerb_ticket</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a> or path, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>If not None, the path to the Kerberos ticket cache.</p> |
| </dd> |
| <dt><strong>extra_conf</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">dict</span></code></a>, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>Extra key/value pairs for configuration; will override any |
| hdfs-site.xml properties.</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.__init__"> |
| <span class="sig-name descname"><span class="pre">__init__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.__init__" title="Permalink to this definition">¶</a></dt> |
| <dd></dd></dl> |
| |
| <p class="rubric">Methods</p> |
| <table class="longtable table autosummary"> |
| <colgroup> |
| <col style="width: 10%" /> |
| <col style="width: 90%" /> |
| </colgroup> |
| <tbody> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.__init__" title="pyarrow.fs.HadoopFileSystem.__init__"><code class="xref py py-obj docutils literal notranslate"><span class="pre">__init__</span></code></a>(*args, **kwargs)</p></td> |
| <td><p></p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.copy_file" title="pyarrow.fs.HadoopFileSystem.copy_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">copy_file</span></code></a>(self, src, dest)</p></td> |
| <td><p>Copy a file.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.create_dir" title="pyarrow.fs.HadoopFileSystem.create_dir"><code class="xref py py-obj docutils literal notranslate"><span class="pre">create_dir</span></code></a>(self, path, *, bool recursive=True)</p></td> |
| <td><p>Create a directory and subdirectories.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.delete_dir" title="pyarrow.fs.HadoopFileSystem.delete_dir"><code class="xref py py-obj docutils literal notranslate"><span class="pre">delete_dir</span></code></a>(self, path)</p></td> |
| <td><p>Delete a directory and its contents, recursively.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.delete_dir_contents" title="pyarrow.fs.HadoopFileSystem.delete_dir_contents"><code class="xref py py-obj docutils literal notranslate"><span class="pre">delete_dir_contents</span></code></a>(self, path, *, ...)</p></td> |
| <td><p>Delete a directory's contents, recursively.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.delete_file" title="pyarrow.fs.HadoopFileSystem.delete_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">delete_file</span></code></a>(self, path)</p></td> |
| <td><p>Delete a file.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.equals" title="pyarrow.fs.HadoopFileSystem.equals"><code class="xref py py-obj docutils literal notranslate"><span class="pre">equals</span></code></a>(self, FileSystem other)</p></td> |
| <td><p></p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.from_uri" title="pyarrow.fs.HadoopFileSystem.from_uri"><code class="xref py py-obj docutils literal notranslate"><span class="pre">from_uri</span></code></a>(uri)</p></td> |
| <td><p>Instantiate HadoopFileSystem object from an URI string.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.get_file_info" title="pyarrow.fs.HadoopFileSystem.get_file_info"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_file_info</span></code></a>(self, paths_or_selector)</p></td> |
| <td><p>Get info for the given files.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.move" title="pyarrow.fs.HadoopFileSystem.move"><code class="xref py py-obj docutils literal notranslate"><span class="pre">move</span></code></a>(self, src, dest)</p></td> |
| <td><p>Move / rename a file or directory.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.normalize_path" title="pyarrow.fs.HadoopFileSystem.normalize_path"><code class="xref py py-obj docutils literal notranslate"><span class="pre">normalize_path</span></code></a>(self, path)</p></td> |
| <td><p>Normalize filesystem path.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.open_append_stream" title="pyarrow.fs.HadoopFileSystem.open_append_stream"><code class="xref py py-obj docutils literal notranslate"><span class="pre">open_append_stream</span></code></a>(self, path[, ...])</p></td> |
| <td><p>Open an output stream for appending.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.open_input_file" title="pyarrow.fs.HadoopFileSystem.open_input_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">open_input_file</span></code></a>(self, path)</p></td> |
| <td><p>Open an input file for random access reading.</p></td> |
| </tr> |
| <tr class="row-even"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.open_input_stream" title="pyarrow.fs.HadoopFileSystem.open_input_stream"><code class="xref py py-obj docutils literal notranslate"><span class="pre">open_input_stream</span></code></a>(self, path[, compression, ...])</p></td> |
| <td><p>Open an input stream for sequential reading.</p></td> |
| </tr> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.open_output_stream" title="pyarrow.fs.HadoopFileSystem.open_output_stream"><code class="xref py py-obj docutils literal notranslate"><span class="pre">open_output_stream</span></code></a>(self, path[, ...])</p></td> |
| <td><p>Open an output stream for sequential writing.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| <p class="rubric">Attributes</p> |
| <table class="longtable table autosummary"> |
| <colgroup> |
| <col style="width: 10%" /> |
| <col style="width: 90%" /> |
| </colgroup> |
| <tbody> |
| <tr class="row-odd"><td><p><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem.type_name" title="pyarrow.fs.HadoopFileSystem.type_name"><code class="xref py py-obj docutils literal notranslate"><span class="pre">type_name</span></code></a></p></td> |
| <td><p>The filesystem's type name.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.copy_file"> |
| <span class="sig-name descname"><span class="pre">copy_file</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">src</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dest</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.copy_file" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Copy a file.</p> |
| <p>If the destination exists and is a directory, an error is returned. |
| Otherwise, it is replaced.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>src</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The path of the file to be copied from.</p> |
| </dd> |
| <dt><strong>dest</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The destination path where the file is copied to.</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.create_dir"> |
| <span class="sig-name descname"><span class="pre">create_dir</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bool</span> <span class="pre">recursive=True</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.create_dir" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Create a directory and subdirectories.</p> |
| <p>This function succeeds if the directory already exists.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The path of the new directory.</p> |
| </dd> |
| <dt><strong>recursive: bool, default True</strong></dt><dd><p>Create nested directories as well.</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.delete_dir"> |
| <span class="sig-name descname"><span class="pre">delete_dir</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.delete_dir" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Delete a directory and its contents, recursively.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The path of the directory to be deleted.</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.delete_dir_contents"> |
| <span class="sig-name descname"><span class="pre">delete_dir_contents</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">bool</span> <span class="pre">accept_root_dir=False</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.delete_dir_contents" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Delete a directory’s contents, recursively.</p> |
| <p>Like delete_dir, but doesn’t delete the directory itself.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The path of the directory to be deleted.</p> |
| </dd> |
| <dt><strong>accept_root_dir</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#bltin-boolean-values" title="(in Python v3.10)"><span class="xref std std-ref">bool</span></a>, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#False" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">False</span></code></a></span></dt><dd><p>Allow deleting the root directory’s contents |
| (if path is empty or “/”)</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.delete_file"> |
| <span class="sig-name descname"><span class="pre">delete_file</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.delete_file" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Delete a file.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The path of the file to be deleted.</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.equals"> |
| <span class="sig-name descname"><span class="pre">equals</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">FileSystem</span> <span class="pre">other</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.equals" title="Permalink to this definition">¶</a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.from_uri"> |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_uri</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">uri</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.from_uri" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Instantiate HadoopFileSystem object from an URI string.</p> |
| <p>The following two calls are equivalent</p> |
| <ul class="simple"> |
| <li><p><code class="docutils literal notranslate"><span class="pre">HadoopFileSystem.from_uri('hdfs://localhost:8020/?user=test&replication=1')</span></code></p></li> |
| <li><p><code class="docutils literal notranslate"><span class="pre">HadoopFileSystem('localhost',</span> <span class="pre">port=8020,</span> <span class="pre">user='test',</span> <span class="pre">replication=1)</span></code></p></li> |
| </ul> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>uri</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>A string URI describing the connection to HDFS. |
| In order to change the user, replication, buffer_size or |
| default_block_size pass the values as query parts.</p> |
| </dd> |
| </dl> |
| </dd> |
| <dt class="field-even">Returns</dt> |
| <dd class="field-even"><dl class="simple"> |
| <dt><a class="reference internal" href="#pyarrow.fs.HadoopFileSystem" title="pyarrow.fs.HadoopFileSystem"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem</span></code></a></dt><dd></dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.get_file_info"> |
| <span class="sig-name descname"><span class="pre">get_file_info</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">paths_or_selector</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.get_file_info" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Get info for the given files.</p> |
| <p>Any symlink is automatically dereferenced, recursively. A non-existing |
| or unreachable file returns a FileStat object and has a FileType of |
| value NotFound. An exception indicates a truly exceptional condition |
| (low-level I/O error, etc.).</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl class="simple"> |
| <dt><strong>paths_or_selector: FileSelector, path-like or list of path-likes</strong></dt><dd><p>Either a selector object, a path-like object or a list of |
| path-like objects. The selector’s base directory will not be |
| part of the results, even if it exists. If it doesn’t exist, |
| use <cite>allow_not_found</cite>.</p> |
| </dd> |
| </dl> |
| </dd> |
| <dt class="field-even">Returns</dt> |
| <dd class="field-even"><dl class="simple"> |
| <dt><a class="reference internal" href="pyarrow.fs.FileInfo.html#pyarrow.fs.FileInfo" title="pyarrow.fs.FileInfo"><code class="xref py py-obj docutils literal notranslate"><span class="pre">FileInfo</span></code></a> or <a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">list</span></code></a> of <a class="reference internal" href="pyarrow.fs.FileInfo.html#pyarrow.fs.FileInfo" title="pyarrow.fs.FileInfo"><code class="xref py py-obj docutils literal notranslate"><span class="pre">FileInfo</span></code></a></dt><dd><p>Single FileInfo object is returned for a single path, otherwise |
| a list of FileInfo objects is returned.</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.move"> |
| <span class="sig-name descname"><span class="pre">move</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">src</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dest</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.move" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Move / rename a file or directory.</p> |
| <p>If the destination exists: |
| - if it is a non-empty directory, an error is returned |
| - otherwise, if it has the same type as the source, it is replaced |
| - otherwise, behavior is unspecified (implementation-dependent).</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>src</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The path of the file or the directory to be moved.</p> |
| </dd> |
| <dt><strong>dest</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The destination path where the file or directory is moved to.</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.normalize_path"> |
| <span class="sig-name descname"><span class="pre">normalize_path</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.normalize_path" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Normalize filesystem path.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The path to normalize</p> |
| </dd> |
| </dl> |
| </dd> |
| <dt class="field-even">Returns</dt> |
| <dd class="field-even"><dl> |
| <dt><strong>normalized_path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The normalized path</p> |
| </dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.open_append_stream"> |
| <span class="sig-name descname"><span class="pre">open_append_stream</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'detect'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffer_size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.open_append_stream" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Open an output stream for appending.</p> |
| <p>If the target doesn’t exist, a new empty file is created.</p> |
| <div class="admonition note"> |
| <p class="admonition-title">Note</p> |
| <p>Some filesystem implementations do not support efficient |
| appending to an existing file, in which case this method will |
| raise NotImplementedError. |
| Consider writing to multiple files (using e.g. the dataset layer) |
| instead.</p> |
| </div> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The source to open for writing.</p> |
| </dd> |
| <dt><strong>compression</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a> optional, default ‘detect’</span></dt><dd><p>The compression algorithm to use for on-the-fly compression. |
| If “detect” and source is a file path, then compression will be |
| chosen based on the file extension. |
| If None, no compression will be applied. Otherwise, a well-known |
| algorithm name must be supplied (e.g. “gzip”).</p> |
| </dd> |
| <dt><strong>buffer_size</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">int</span></code></a> optional, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>If None or 0, no buffering will happen. Otherwise the size of the |
| temporary write buffer.</p> |
| </dd> |
| <dt><strong>metadata</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">dict</span></code></a> optional, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>If not None, a mapping of string keys to string values. |
| Some filesystems support storing metadata along the file |
| (such as “Content-Type”). |
| Unsupported metadata keys will be ignored.</p> |
| </dd> |
| </dl> |
| </dd> |
| <dt class="field-even">Returns</dt> |
| <dd class="field-even"><dl> |
| <dt><strong>stream</strong><span class="classifier"><a class="reference internal" href="pyarrow.NativeFile.html#pyarrow.NativeFile" title="pyarrow.NativeFile"><code class="xref py py-obj docutils literal notranslate"><span class="pre">NativeFile</span></code></a></span></dt><dd></dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.open_input_file"> |
| <span class="sig-name descname"><span class="pre">open_input_file</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.open_input_file" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Open an input file for random access reading.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The source to open for reading.</p> |
| </dd> |
| </dl> |
| </dd> |
| <dt class="field-even">Returns</dt> |
| <dd class="field-even"><dl> |
| <dt><strong>stram</strong><span class="classifier"><a class="reference internal" href="pyarrow.NativeFile.html#pyarrow.NativeFile" title="pyarrow.NativeFile"><code class="xref py py-obj docutils literal notranslate"><span class="pre">NativeFile</span></code></a></span></dt><dd></dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.open_input_stream"> |
| <span class="sig-name descname"><span class="pre">open_input_stream</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'detect'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffer_size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.open_input_stream" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Open an input stream for sequential reading.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>source</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The source to open for reading.</p> |
| </dd> |
| <dt><strong>compression</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a> optional, default ‘detect’</span></dt><dd><p>The compression algorithm to use for on-the-fly decompression. |
| If “detect” and source is a file path, then compression will be |
| chosen based on the file extension. |
| If None, no compression will be applied. Otherwise, a well-known |
| algorithm name must be supplied (e.g. “gzip”).</p> |
| </dd> |
| <dt><strong>buffer_size</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">int</span></code></a> optional, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>If None or 0, no buffering will happen. Otherwise the size of the |
| temporary read buffer.</p> |
| </dd> |
| </dl> |
| </dd> |
| <dt class="field-even">Returns</dt> |
| <dd class="field-even"><dl> |
| <dt><strong>stream</strong><span class="classifier"><a class="reference internal" href="pyarrow.NativeFile.html#pyarrow.NativeFile" title="pyarrow.NativeFile"><code class="xref py py-obj docutils literal notranslate"><span class="pre">NativeFile</span></code></a></span></dt><dd></dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.open_output_stream"> |
| <span class="sig-name descname"><span class="pre">open_output_stream</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">self</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'detect'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffer_size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.open_output_stream" title="Permalink to this definition">¶</a></dt> |
| <dd><p>Open an output stream for sequential writing.</p> |
| <p>If the target already exists, existing data is truncated.</p> |
| <dl class="field-list"> |
| <dt class="field-odd">Parameters</dt> |
| <dd class="field-odd"><dl> |
| <dt><strong>path</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a></span></dt><dd><p>The source to open for writing.</p> |
| </dd> |
| <dt><strong>compression</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">str</span></code></a> optional, default ‘detect’</span></dt><dd><p>The compression algorithm to use for on-the-fly compression. |
| If “detect” and source is a file path, then compression will be |
| chosen based on the file extension. |
| If None, no compression will be applied. Otherwise, a well-known |
| algorithm name must be supplied (e.g. “gzip”).</p> |
| </dd> |
| <dt><strong>buffer_size</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">int</span></code></a> optional, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>If None or 0, no buffering will happen. Otherwise the size of the |
| temporary write buffer.</p> |
| </dd> |
| <dt><strong>metadata</strong><span class="classifier"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">dict</span></code></a> optional, default <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.10)"><code class="docutils literal notranslate"><span class="pre">None</span></code></a></span></dt><dd><p>If not None, a mapping of string keys to string values. |
| Some filesystems support storing metadata along the file |
| (such as “Content-Type”). |
| Unsupported metadata keys will be ignored.</p> |
| </dd> |
| </dl> |
| </dd> |
| <dt class="field-even">Returns</dt> |
| <dd class="field-even"><dl> |
| <dt><strong>stream</strong><span class="classifier"><a class="reference internal" href="pyarrow.NativeFile.html#pyarrow.NativeFile" title="pyarrow.NativeFile"><code class="xref py py-obj docutils literal notranslate"><span class="pre">NativeFile</span></code></a></span></dt><dd></dd> |
| </dl> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="pyarrow.fs.HadoopFileSystem.type_name"> |
| <span class="sig-name descname"><span class="pre">type_name</span></span><a class="headerlink" href="#pyarrow.fs.HadoopFileSystem.type_name" title="Permalink to this definition">¶</a></dt> |
| <dd><p>The filesystem’s type name.</p> |
| </dd></dl> |
| |
| </dd></dl> |
| |
| </div> |
| |
| |
| </div> |
| |
| |
| <!-- Previous / next buttons --> |
| <div class='prev-next-area'> |
| <a class='left-prev' id="prev-link" href="pyarrow.fs.S3FileSystem.html" title="previous page"> |
| <i class="fas fa-angle-left"></i> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">previous</p> |
| <p class="prev-next-title">pyarrow.fs.S3FileSystem</p> |
| </div> |
| </a> |
| <a class='right-next' id="next-link" href="pyarrow.fs.SubTreeFileSystem.html" title="next page"> |
| <div class="prev-next-info"> |
| <p class="prev-next-subtitle">next</p> |
| <p class="prev-next-title">pyarrow.fs.SubTreeFileSystem</p> |
| </div> |
| <i class="fas fa-angle-right"></i> |
| </a> |
| </div> |
| |
| </main> |
| |
| |
| </div> |
| </div> |
| |
| <script src="../../_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"></script> |
| <footer class="footer mt-5 mt-md-0"> |
| <div class="container"> |
| |
| <div class="footer-item"> |
| <p class="copyright"> |
| © Copyright 2016-2022 Apache Software Foundation.<br> |
| </p> |
| </div> |
| |
| <div class="footer-item"> |
| <p class="sphinx-version"> |
| Created using <a href="http://sphinx-doc.org/">Sphinx</a> 4.4.0.<br> |
| </p> |
| </div> |
| |
| </div> |
| </footer> |
| <script type="text/javascript" src="/docs/_static/versionwarning.js"></script> </body> |
| </html> |
