docs/4.0/r/reference/FileFormat.html - arrow-site - Git at Google

 <!-- Generated by pkgdown: do not edit by hand -->
 <!DOCTYPE html>
 <html lang="en">
   <head>
   <meta charset="utf-8">
 <meta http-equiv="X-UA-Compatible" content="IE=edge">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">

 <title>Dataset file formats — FileFormat • Arrow R Package</title>


 <!-- jquery -->
 <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
 <!-- Bootstrap -->
 <link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />


 <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>

 <!-- bootstrap-toc -->
 <link rel="stylesheet" href="../bootstrap-toc.css">
 <script src="../bootstrap-toc.js"></script>

 <!-- Font Awesome icons -->
 <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
 <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />

 <!-- clipboard.js -->
 <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>

 <!-- headroom.js -->
 <script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>

 <!-- pkgdown -->
 <link href="../pkgdown.css" rel="stylesheet">
 <script src="../pkgdown.js"></script>


   <script src="../extra.js"></script>

 <meta property="og:title" content="Dataset file formats — FileFormat" />
 <meta property="og:description" content="A FileFormat holds information about how to read and parse the files
 included in a Dataset. There are subclasses corresponding to the supported
 file formats (ParquetFileFormat and IpcFileFormat)." />


 <!-- mathjax -->
 <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>

 <!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
 <![endif]-->

     <!-- Matomo -->
     <script>
       var _paq = window._paq = window._paq || [];
       /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
       _paq.push(["setDoNotTrack", true]);
       _paq.push(["disableCookies"]);
       _paq.push(['trackPageView']);
       _paq.push(['enableLinkTracking']);
       (function() {
         var u="https://analytics.apache.org/";
         _paq.push(['setTrackerUrl', u+'matomo.php']);
         _paq.push(['setSiteId', '20']);
         var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
         g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
       })();
     </script>
     <!-- End Matomo Code -->

 </head>

   <body data-spy="scroll" data-target="#toc">
     <div class="container template-reference-topic">
       <header>
       <div class="navbar navbar-default navbar-fixed-top" role="navigation">
   <div class="container">
     <div class="navbar-header">
       <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
         <span class="sr-only">Toggle navigation</span>
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </button>
       <span class="navbar-brand">
         <a class="navbar-link" href="../index.html">Arrow R Package</a>
         <span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.0.1</span>
       </span>
     </div>

     <div id="navbar" class="navbar-collapse collapse">
       <ul class="nav navbar-nav">
         <li>
   <a href="https://arrow.apache.org/">❯❯❯</a>
 </li>
 <li>
   <a href="../articles/arrow.html">Get started</a>
 </li>
 <li>
   <a href="../reference/index.html">Reference</a>
 </li>
 <li class="dropdown">
   <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
     Articles

     <span class="caret"></span>
   </a>
   <ul class="dropdown-menu" role="menu">
     <li>
       <a href="../articles/install.html">Installing the Arrow Package on Linux</a>
     </li>
     <li>
       <a href="../articles/dataset.html">Working with Arrow Datasets and dplyr</a>
     </li>
     <li>
       <a href="../articles/fs.html">Working with Cloud Storage (S3)</a>
     </li>
     <li>
       <a href="../articles/python.html">Apache Arrow in Python and R with reticulate</a>
     </li>
     <li>
       <a href="../articles/flight.html">Connecting to Flight RPC Servers</a>
     </li>
     <li>
       <a href="../articles/developing.html">Arrow R Developer Guide</a>
     </li>
   </ul>
 </li>
 <li>
   <a href="../news/index.html">Changelog</a>
 </li>
 <li class="dropdown">
   <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
     Project docs

     <span class="caret"></span>
   </a>
   <ul class="dropdown-menu" role="menu">
     <li>
       <a href="https://arrow.apache.org/docs/format/README.html">Specification</a>
     </li>
     <li>
       <a href="https://arrow.apache.org/docs/c_glib">C GLib</a>
     </li>
     <li>
       <a href="https://arrow.apache.org/docs/cpp">C++</a>
     </li>
     <li>
       <a href="https://arrow.apache.org/docs/java">Java</a>
     </li>
     <li>
       <a href="https://arrow.apache.org/docs/js">JavaScript</a>
     </li>
     <li>
       <a href="https://arrow.apache.org/docs/python">Python</a>
     </li>
     <li>
       <a href="../index.html">R</a>
     </li>
   </ul>
 </li>
       </ul>
       <ul class="nav navbar-nav navbar-right">

       </ul>

     </div><!--/.nav-collapse -->
   </div><!--/.container -->
 </div><!--/.navbar -->


       </header>

 <div class="row">
   <div class="col-md-9 contents">
     <div class="page-header">
     <h1>Dataset file formats</h1>
     <small class="dont-index">Source: <a href='https://github.com/apache/arrow/blob/master/r/R/dataset-format.R'><code>R/dataset-format.R</code></a></small>
     <div class="hidden name"><code>FileFormat.Rd</code></div>
     </div>

     <div class="ref-description">
     <p>A <code>FileFormat</code> holds information about how to read and parse the files
 included in a <code>Dataset</code>. There are subclasses corresponding to the supported
 file formats (<code>ParquetFileFormat</code> and <code>IpcFileFormat</code>).</p>
     </div>


     <h2 class="hasAnchor" id="factory"><a class="anchor" href="#factory"></a>Factory</h2>


 <p><code>FileFormat$create()</code> takes the following arguments:</p><ul>
 <li><p><code>format</code>: A string identifier of the file format. Currently supported values:</p><ul>
 <li><p>"parquet"</p></li>
 <li><p>"ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
 only version 2 files are supported</p></li>
 <li><p>"csv"/"text", aliases for the same thing (because comma is the default
 delimiter for text files</p></li>
 <li><p>"tsv", equivalent to passing <code>format = "text", delimiter = "\t"</code></p></li>
 </ul></li>
 <li><p><code>...</code>: Additional format-specific options</p>
 <p>`format = "parquet"``:</p><ul>
 <li><p><code>use_buffered_stream</code>: Read files through buffered input streams rather than
 loading entire row groups at once. This may be enabled
 to reduce memory overhead. Disabled by default.</p></li>
 <li><p><code>buffer_size</code>: Size of buffered stream, if enabled. Default is 8KB.</p></li>
 <li><p><code>dict_columns</code>: Names of columns which should be read as dictionaries.</p></li>
 </ul>

 <p><code>format = "text"</code>: see <a href='CsvReadOptions.html'>CsvParseOptions</a>. Note that you can specify them either
 with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
 <code>readr</code>-style naming used in <code><a href='read_delim_arrow.html'>read_csv_arrow()</a></code> ("delim", "quote", etc.).
 Not all <code>readr</code> options are currently supported; please file an issue if
 you encounter one that <code>arrow</code> should support. Also, the following options are
 supported. From <a href='CsvReadOptions.html'>CsvReadOptions</a>:</p><ul>
 <li><p><code>skip_rows</code></p></li>
 <li><p><code>column_names</code></p></li>
 <li><p><code>autogenerate_column_names</code>
 From <a href='FragmentScanOptions.html'>CsvFragmentScanOptions</a> (these values can be overridden at scan time):</p></li>
 <li><p><code>convert_options</code>: a <a href='CsvReadOptions.html'>CsvConvertOptions</a></p></li>
 <li><p><code>block_size</code></p></li>
 </ul></li>
 </ul>

 <p>It returns the appropriate subclass of <code>FileFormat</code> (e.g. <code>ParquetFileFormat</code>)</p>

   </div>
   <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
     <nav id="toc" data-toggle="toc" class="sticky-top">
       <h2 data-toc-skip>Contents</h2>
     </nav>
   </div>
 </div>


       <footer>
       <div class="copyright">
   <p>Developed by Neal Richardson, Ian Cook, Jonathan Keane, Romain François, Jeroen Ooms, Apache Arrow.</p>
 </div>

 <div class="pkgdown">
   <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
 </div>

       </footer>
    </div>


 <script type="text/javascript" src="/docs/_static/versionwarning.js"></script>  </body>
 </html>
	<!-- Generated by pkgdown: do not edit by hand -->
	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="utf-8">
	<meta http-equiv="X-UA-Compatible" content="IE=edge">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">

	<title>Dataset file formats — FileFormat • Arrow R Package</title>


	<!-- jquery -->
	<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
	<!-- Bootstrap -->
	<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/cosmo/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />


	<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>

	<!-- bootstrap-toc -->
	<link rel="stylesheet" href="../bootstrap-toc.css">
	<script src="../bootstrap-toc.js"></script>

	<!-- Font Awesome icons -->
	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
	<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />

	<!-- clipboard.js -->
	<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>

	<!-- headroom.js -->
	<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
	<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>

	<!-- pkgdown -->
	<link href="../pkgdown.css" rel="stylesheet">
	<script src="../pkgdown.js"></script>




	<script src="../extra.js"></script>

	<meta property="og:title" content="Dataset file formats — FileFormat" />
	<meta property="og:description" content="A FileFormat holds information about how to read and parse the files
	included in a Dataset. There are subclasses corresponding to the supported
	file formats (ParquetFileFormat and IpcFileFormat)." />




	<!-- mathjax -->
	<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
	<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>

	<!--[if lt IE 9]>
	<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
	<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
	<![endif]-->

	<!-- Matomo -->
	<script>
	var _paq = window._paq = window._paq \|\| [];
	/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
	_paq.push(["setDoNotTrack", true]);
	_paq.push(["disableCookies"]);
	_paq.push(['trackPageView']);
	_paq.push(['enableLinkTracking']);
	(function() {
	var u="https://analytics.apache.org/";
	_paq.push(['setTrackerUrl', u+'matomo.php']);
	_paq.push(['setSiteId', '20']);
	var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
	g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
	})();
	</script>
	<!-- End Matomo Code -->

	</head>

	<body data-spy="scroll" data-target="#toc">
	<div class="container template-reference-topic">
	<header>
	<div class="navbar navbar-default navbar-fixed-top" role="navigation">
	<div class="container">
	<div class="navbar-header">
	<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
	<span class="sr-only">Toggle navigation</span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	</button>
	<span class="navbar-brand">
	<a class="navbar-link" href="../index.html">Arrow R Package</a>
	<span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">4.0.1</span>
	</span>
	</div>

	<div id="navbar" class="navbar-collapse collapse">
	<ul class="nav navbar-nav">
	<li>
	<a href="https://arrow.apache.org/">❯❯❯</a>
	</li>
	<li>
	<a href="../articles/arrow.html">Get started</a>
	</li>
	<li>
	<a href="../reference/index.html">Reference</a>
	</li>
	<li class="dropdown">
	<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
	Articles

	<span class="caret"></span>
	</a>
	<ul class="dropdown-menu" role="menu">
	<li>
	<a href="../articles/install.html">Installing the Arrow Package on Linux</a>
	</li>
	<li>
	<a href="../articles/dataset.html">Working with Arrow Datasets and dplyr</a>
	</li>
	<li>
	<a href="../articles/fs.html">Working with Cloud Storage (S3)</a>
	</li>
	<li>
	<a href="../articles/python.html">Apache Arrow in Python and R with reticulate</a>
	</li>
	<li>
	<a href="../articles/flight.html">Connecting to Flight RPC Servers</a>
	</li>
	<li>
	<a href="../articles/developing.html">Arrow R Developer Guide</a>
	</li>
	</ul>
	</li>
	<li>
	<a href="../news/index.html">Changelog</a>
	</li>
	<li class="dropdown">
	<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
	Project docs

	<span class="caret"></span>
	</a>
	<ul class="dropdown-menu" role="menu">
	<li>
	<a href="https://arrow.apache.org/docs/format/README.html">Specification</a>
	</li>
	<li>
	<a href="https://arrow.apache.org/docs/c_glib">C GLib</a>
	</li>
	<li>
	<a href="https://arrow.apache.org/docs/cpp">C++</a>
	</li>
	<li>
	<a href="https://arrow.apache.org/docs/java">Java</a>
	</li>
	<li>
	<a href="https://arrow.apache.org/docs/js">JavaScript</a>
	</li>
	<li>
	<a href="https://arrow.apache.org/docs/python">Python</a>
	</li>
	<li>
	<a href="../index.html">R</a>
	</li>
	</ul>
	</li>
	</ul>
	<ul class="nav navbar-nav navbar-right">

	</ul>

	</div><!--/.nav-collapse -->
	</div><!--/.container -->
	</div><!--/.navbar -->



	</header>

	<div class="row">
	<div class="col-md-9 contents">
	<div class="page-header">
	<h1>Dataset file formats</h1>
	<small class="dont-index">Source: <a href='https://github.com/apache/arrow/blob/master/r/R/dataset-format.R'><code>R/dataset-format.R</code></a></small>
	<div class="hidden name"><code>FileFormat.Rd</code></div>
	</div>

	<div class="ref-description">
	<p>A <code>FileFormat</code> holds information about how to read and parse the files
	included in a <code>Dataset</code>. There are subclasses corresponding to the supported
	file formats (<code>ParquetFileFormat</code> and <code>IpcFileFormat</code>).</p>
	</div>



	<h2 class="hasAnchor" id="factory"><a class="anchor" href="#factory"></a>Factory</h2>



	<p><code>FileFormat$create()</code> takes the following arguments:</p><ul>
	<li><p><code>format</code>: A string identifier of the file format. Currently supported values:</p><ul>
	<li><p>"parquet"</p></li>
	<li><p>"ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
	only version 2 files are supported</p></li>
	<li><p>"csv"/"text", aliases for the same thing (because comma is the default
	delimiter for text files</p></li>
	<li><p>"tsv", equivalent to passing <code>format = "text", delimiter = "\t"</code></p></li>
	</ul></li>
	<li><p><code>...</code>: Additional format-specific options</p>
	<p>`format = "parquet"``:</p><ul>
	<li><p><code>use_buffered_stream</code>: Read files through buffered input streams rather than
	loading entire row groups at once. This may be enabled
	to reduce memory overhead. Disabled by default.</p></li>
	<li><p><code>buffer_size</code>: Size of buffered stream, if enabled. Default is 8KB.</p></li>
	<li><p><code>dict_columns</code>: Names of columns which should be read as dictionaries.</p></li>
	</ul>

	<p><code>format = "text"</code>: see <a href='CsvReadOptions.html'>CsvParseOptions</a>. Note that you can specify them either
	with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
	<code>readr</code>-style naming used in <code><a href='read_delim_arrow.html'>read_csv_arrow()</a></code> ("delim", "quote", etc.).
	Not all <code>readr</code> options are currently supported; please file an issue if
	you encounter one that <code>arrow</code> should support. Also, the following options are
	supported. From <a href='CsvReadOptions.html'>CsvReadOptions</a>:</p><ul>
	<li><p><code>skip_rows</code></p></li>
	<li><p><code>column_names</code></p></li>
	<li><p><code>autogenerate_column_names</code>
	From <a href='FragmentScanOptions.html'>CsvFragmentScanOptions</a> (these values can be overridden at scan time):</p></li>
	<li><p><code>convert_options</code>: a <a href='CsvReadOptions.html'>CsvConvertOptions</a></p></li>
	<li><p><code>block_size</code></p></li>
	</ul></li>
	</ul>

	<p>It returns the appropriate subclass of <code>FileFormat</code> (e.g. <code>ParquetFileFormat</code>)</p>

	</div>
	<div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
	<nav id="toc" data-toggle="toc" class="sticky-top">
	<h2 data-toc-skip>Contents</h2>
	</nav>
	</div>
	</div>


	<footer>
	<div class="copyright">
	<p>Developed by Neal Richardson, Ian Cook, Jonathan Keane, Romain François, Jeroen Ooms, Apache Arrow.</p>
	</div>

	<div class="pkgdown">
	<p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a> 1.6.1.</p>
	</div>

	</footer>
	</div>




	<script type="text/javascript" src="/docs/_static/versionwarning.js"></script> </body>
	</html>