blob: fee039d2661b39447d0459476847eb60a64f54b5 [file] [log] [blame]
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Filesystem Interface (legacy) &mdash; Apache Arrow v3.0.0</title>
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/theme_overrides.css" type="text/css" />
<!--[if lt IE 9]>
<script src="../_static/js/html5shiv.min.js"></script>
<![endif]-->
<script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script src="../_static/jquery.js"></script>
<script src="../_static/underscore.js"></script>
<script src="../_static/doctools.js"></script>
<script src="../_static/language_data.js"></script>
<script type="text/javascript" src="../_static/js/theme.js"></script>
<link rel="canonical" href="https://arrow.apache.org/docs/python/filesystems_deprecated.html" />
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="pyarrow.hdfs.connect" href="generated/pyarrow.hdfs.connect.html" />
<link rel="prev" title="Filesystem Interface" href="filesystems.html" />
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["setDoNotTrack", true]);
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '20']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../index.html" class="icon icon-home"> Apache Arrow
</a>
<div class="version">
3.0.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<p class="caption"><span class="caption-text">Specifications and Protocols</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../format/Versioning.html">Format Versioning and Stability</a></li>
<li class="toctree-l1"><a class="reference internal" href="../format/Columnar.html">Arrow Columnar Format</a></li>
<li class="toctree-l1"><a class="reference internal" href="../format/Flight.html">Arrow Flight RPC</a></li>
<li class="toctree-l1"><a class="reference internal" href="../format/Integration.html">Integration Testing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../format/CDataInterface.html">The Arrow C data interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../format/CStreamInterface.html">The Arrow C stream interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../format/Other.html">Other Data Structures</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../status.html">Implementation Status</a></li>
<li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/docs/c_glib/">C/GLib</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cpp/index.html">C++</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/master/csharp/README.md">C#</a></li>
<li class="toctree-l1"><a class="reference external" href="https://godoc.org/github.com/apache/arrow/go/arrow">Go</a></li>
<li class="toctree-l1"><a class="reference internal" href="../java/index.html">Java</a></li>
<li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/docs/js/">JavaScript</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/master/julia/Arrow/README.md">Julia</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/master/matlab/README.md">MATLAB</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html">Python</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="install.html">Installing PyArrow</a></li>
<li class="toctree-l2"><a class="reference internal" href="memory.html">Memory and IO Interfaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="data.html">Data Types and In-Memory Data Model</a></li>
<li class="toctree-l2"><a class="reference internal" href="compute.html">Compute Functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="ipc.html">Streaming, Serialization, and IPC</a></li>
<li class="toctree-l2"><a class="reference internal" href="filesystems.html">Filesystem Interface</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">Filesystem Interface (legacy)</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#hadoop-file-system-hdfs">Hadoop File System (HDFS)</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#hdfs-api">HDFS API</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="plasma.html">The Plasma In-Memory Object Store</a></li>
<li class="toctree-l2"><a class="reference internal" href="numpy.html">NumPy Integration</a></li>
<li class="toctree-l2"><a class="reference internal" href="pandas.html">Pandas Integration</a></li>
<li class="toctree-l2"><a class="reference internal" href="timestamps.html">Timestamps</a></li>
<li class="toctree-l2"><a class="reference internal" href="csv.html">Reading CSV files</a></li>
<li class="toctree-l2"><a class="reference internal" href="feather.html">Feather File Format</a></li>
<li class="toctree-l2"><a class="reference internal" href="json.html">Reading JSON files</a></li>
<li class="toctree-l2"><a class="reference internal" href="parquet.html">Reading and Writing the Apache Parquet Format</a></li>
<li class="toctree-l2"><a class="reference internal" href="dataset.html">Tabular Datasets</a></li>
<li class="toctree-l2"><a class="reference internal" href="cuda.html">CUDA Integration</a></li>
<li class="toctree-l2"><a class="reference internal" href="extending_types.html">Extending pyarrow</a></li>
<li class="toctree-l2"><a class="reference internal" href="extending.html">Using pyarrow from C++ and Cython Code</a></li>
<li class="toctree-l2"><a class="reference internal" href="api.html">API Reference</a></li>
<li class="toctree-l2"><a class="reference internal" href="getting_involved.html">Getting Involved</a></li>
<li class="toctree-l2"><a class="reference internal" href="benchmarks.html">Benchmarks</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference external" href="https://arrow.apache.org/docs/r/">R</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/apache/arrow/blob/master/ruby/README.md">Ruby</a></li>
<li class="toctree-l1"><a class="reference external" href="https://docs.rs/crate/arrow/">Rust</a></li>
</ul>
<p class="caption"><span class="caption-text">Development</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../developers/contributing.html">Contributing to Apache Arrow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../developers/cpp/index.html">C++ Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="../developers/python.html">Python Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="../developers/archery.html">Daily Development using Archery</a></li>
<li class="toctree-l1"><a class="reference internal" href="../developers/crossbow.html">Packaging and Testing with Crossbow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../developers/docker.html">Running Docker Builds</a></li>
<li class="toctree-l1"><a class="reference internal" href="../developers/benchmarks.html">Benchmarks</a></li>
<li class="toctree-l1"><a class="reference internal" href="../developers/documentation.html">Building the Documentation</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../index.html">Apache Arrow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
<li><a href="index.html">Python bindings</a> &raquo;</li>
<li>Filesystem Interface (legacy)</li>
<li class="wy-breadcrumbs-aside">
<a href="../_sources/python/filesystems_deprecated.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="filesystem-interface-legacy">
<h1>Filesystem Interface (legacy)<a class="headerlink" href="#filesystem-interface-legacy" title="Permalink to this headline"></a></h1>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This section documents the deprecated filesystem layer. You should
use the <a class="reference internal" href="filesystems.html#filesystem"><span class="std std-ref">new filesystem layer</span></a> instead.</p>
</div>
<div class="section" id="hadoop-file-system-hdfs">
<span id="hdfs"></span><h2>Hadoop File System (HDFS)<a class="headerlink" href="#hadoop-file-system-hdfs" title="Permalink to this headline"></a></h2>
<p>PyArrow comes with bindings to a C++-based interface to the Hadoop File
System. You connect like so:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span>
<span class="n">fs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">hdfs</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span> <span class="n">kerb_ticket</span><span class="o">=</span><span class="n">ticket_cache_path</span><span class="p">)</span>
<span class="k">with</span> <span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="c1"># Do something with f</span>
</pre></div>
</div>
<p>By default, <code class="docutils literal notranslate"><span class="pre">pyarrow.hdfs.HadoopFileSystem</span></code> uses libhdfs, a JNI-based
interface to the Java Hadoop client. This library is loaded <strong>at runtime</strong>
(rather than at link / library load time, since the library may not be in your
LD_LIBRARY_PATH), and relies on some environment variables.</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">HADOOP_HOME</span></code>: the root of your installed Hadoop distribution. Often has
<cite>lib/native/libhdfs.so</cite>.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">JAVA_HOME</span></code>: the location of your Java SDK installation.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">ARROW_LIBHDFS_DIR</span></code> (optional): explicit location of <code class="docutils literal notranslate"><span class="pre">libhdfs.so</span></code> if it is
installed somewhere other than <code class="docutils literal notranslate"><span class="pre">$HADOOP_HOME/lib/native</span></code>.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">CLASSPATH</span></code>: must contain the Hadoop jars. You can set these using:</p></li>
</ul>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span> <span class="nv">CLASSPATH</span><span class="o">=</span><span class="sb">`</span><span class="nv">$HADOOP_HOME</span>/bin/hdfs classpath --glob<span class="sb">`</span>
</pre></div>
</div>
<p>If <code class="docutils literal notranslate"><span class="pre">CLASSPATH</span></code> is not set, then it will be set automatically if the
<code class="docutils literal notranslate"><span class="pre">hadoop</span></code> executable is in your system path, or if <code class="docutils literal notranslate"><span class="pre">HADOOP_HOME</span></code> is set.</p>
<p>You can also use libhdfs3, a thirdparty C++ library for HDFS from Pivotal Labs:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">fs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">hdfs</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span> <span class="n">kerb_ticket</span><span class="o">=</span><span class="n">ticket_cache_path</span><span class="p">,</span>
<span class="n">driver</span><span class="o">=</span><span class="s1">&#39;libhdfs3&#39;</span><span class="p">)</span>
</pre></div>
</div>
<div class="section" id="hdfs-api">
<h3>HDFS API<a class="headerlink" href="#hdfs-api" title="Permalink to this headline"></a></h3>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.hdfs.connect.html#pyarrow.hdfs.connect" title="pyarrow.hdfs.connect"><code class="xref py py-obj docutils literal notranslate"><span class="pre">hdfs.connect</span></code></a>([host, port, user, …])</p></td>
<td><p>Connect to an HDFS cluster.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.cat.html#pyarrow.HadoopFileSystem.cat" title="pyarrow.HadoopFileSystem.cat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.cat</span></code></a>(path)</p></td>
<td><p>Return contents of file as a bytes object.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.chmod.html#pyarrow.HadoopFileSystem.chmod" title="pyarrow.HadoopFileSystem.chmod"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.chmod</span></code></a>(self, path, mode)</p></td>
<td><p>Change file permissions</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.chown.html#pyarrow.HadoopFileSystem.chown" title="pyarrow.HadoopFileSystem.chown"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.chown</span></code></a>(self, path[, owner, …])</p></td>
<td><p>Change file permissions</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.delete.html#pyarrow.HadoopFileSystem.delete" title="pyarrow.HadoopFileSystem.delete"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.delete</span></code></a>(path[, recursive])</p></td>
<td><p>Delete the indicated file or directory.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.df.html#pyarrow.HadoopFileSystem.df" title="pyarrow.HadoopFileSystem.df"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.df</span></code></a>(self)</p></td>
<td><p>Return free space on disk, like the UNIX df command</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.disk_usage.html#pyarrow.HadoopFileSystem.disk_usage" title="pyarrow.HadoopFileSystem.disk_usage"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.disk_usage</span></code></a>(path)</p></td>
<td><p>Compute bytes used by all contents under indicated path in file tree.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.download.html#pyarrow.HadoopFileSystem.download" title="pyarrow.HadoopFileSystem.download"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.download</span></code></a>(self, path, stream)</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.exists.html#pyarrow.HadoopFileSystem.exists" title="pyarrow.HadoopFileSystem.exists"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.exists</span></code></a>(path)</p></td>
<td><p>Return True if path exists.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.get_capacity.html#pyarrow.HadoopFileSystem.get_capacity" title="pyarrow.HadoopFileSystem.get_capacity"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.get_capacity</span></code></a>(self)</p></td>
<td><p>Get reported total capacity of file system</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.get_space_used.html#pyarrow.HadoopFileSystem.get_space_used" title="pyarrow.HadoopFileSystem.get_space_used"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.get_space_used</span></code></a>(self)</p></td>
<td><p>Get space used on file system</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.info.html#pyarrow.HadoopFileSystem.info" title="pyarrow.HadoopFileSystem.info"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.info</span></code></a>(self, path)</p></td>
<td><p>Return detailed HDFS information for path</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.ls.html#pyarrow.HadoopFileSystem.ls" title="pyarrow.HadoopFileSystem.ls"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.ls</span></code></a>(path[, detail])</p></td>
<td><p>Retrieve directory contents and metadata, if requested.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.mkdir.html#pyarrow.HadoopFileSystem.mkdir" title="pyarrow.HadoopFileSystem.mkdir"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.mkdir</span></code></a>(path, **kwargs)</p></td>
<td><p>Create directory in HDFS.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.open.html#pyarrow.HadoopFileSystem.open" title="pyarrow.HadoopFileSystem.open"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.open</span></code></a>(self, path[, mode, …])</p></td>
<td><p>Open HDFS file for reading or writing</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.rename.html#pyarrow.HadoopFileSystem.rename" title="pyarrow.HadoopFileSystem.rename"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.rename</span></code></a>(path, new_path)</p></td>
<td><p>Rename file, like UNIX mv command.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.rm.html#pyarrow.HadoopFileSystem.rm" title="pyarrow.HadoopFileSystem.rm"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.rm</span></code></a>(path[, recursive])</p></td>
<td><p>Alias for FileSystem.delete.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.upload.html#pyarrow.HadoopFileSystem.upload" title="pyarrow.HadoopFileSystem.upload"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HadoopFileSystem.upload</span></code></a>(self, path, stream)</p></td>
<td><p>Upload file-like object to HDFS path</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/pyarrow.HdfsFile.html#pyarrow.HdfsFile" title="pyarrow.HdfsFile"><code class="xref py py-obj docutils literal notranslate"><span class="pre">HdfsFile</span></code></a></p></td>
<td><p></p></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="generated/pyarrow.hdfs.connect.html" class="btn btn-neutral float-right" title="pyarrow.hdfs.connect" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
<a href="filesystems.html" class="btn btn-neutral float-left" title="Filesystem Interface" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&#169; Copyright 2016-2019 Apache Software Foundation.
</p>
</div>
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
<script type="text/javascript" src="/docs/_static/versionwarning.js"></script></body>
</html>