site/docs/2.4.5/sql-data-sources-orc.html - spark-website - Git at Google


 <!DOCTYPE html>
 <!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
 <!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
 <!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
     <head>
         <meta charset="utf-8">
         <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
         <title>ORC Files - Spark 2.4.5 Documentation</title>


         <link rel="stylesheet" href="css/bootstrap.min.css">
         <style>
             body {
                 padding-top: 60px;
                 padding-bottom: 40px;
             }
         </style>
         <meta name="viewport" content="width=device-width">
         <link rel="stylesheet" href="css/bootstrap-responsive.min.css">
         <link rel="stylesheet" href="css/main.css">

         <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>

         <link rel="stylesheet" href="css/pygments-default.css">


         <!-- Google analytics script -->
         <script type="text/javascript">
           var _gaq = _gaq || [];
           _gaq.push(['_setAccount', 'UA-32518208-2']);
           _gaq.push(['_trackPageview']);

           (function() {
             var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
             ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
             var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
           })();
         </script>


     </head>
     <body>
         <!--[if lt IE 7]>
             <p class="chromeframe">You are using an outdated browser. <a href="https://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p>
         <![endif]-->

         <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->

         <div class="navbar navbar-fixed-top" id="topbar">
             <div class="navbar-inner">
                 <div class="container">
                     <div class="brand"><a href="index.html">
                       <img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">2.4.5</span>
                     </div>
                     <ul class="nav">
                         <!--TODO(andyk): Add class="active" attribute to li some how.-->
                         <li><a href="index.html">Overview</a></li>

                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">Programming Guides<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="quick-start.html">Quick Start</a></li>
                                 <li><a href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a></li>
                                 <li><a href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a></li>
                                 <li><a href="structured-streaming-programming-guide.html">Structured Streaming</a></li>
                                 <li><a href="streaming-programming-guide.html">Spark Streaming (DStreams)</a></li>
                                 <li><a href="ml-guide.html">MLlib (Machine Learning)</a></li>
                                 <li><a href="graphx-programming-guide.html">GraphX (Graph Processing)</a></li>
                                 <li><a href="sparkr.html">SparkR (R on Spark)</a></li>
                             </ul>
                         </li>

                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="api/scala/index.html#org.apache.spark.package">Scala</a></li>
                                 <li><a href="api/java/index.html">Java</a></li>
                                 <li><a href="api/python/index.html">Python</a></li>
                                 <li><a href="api/R/index.html">R</a></li>
                                 <li><a href="api/sql/index.html">SQL, Built-in Functions</a></li>
                             </ul>
                         </li>

                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">Deploying<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="cluster-overview.html">Overview</a></li>
                                 <li><a href="submitting-applications.html">Submitting Applications</a></li>
                                 <li class="divider"></li>
                                 <li><a href="spark-standalone.html">Spark Standalone</a></li>
                                 <li><a href="running-on-mesos.html">Mesos</a></li>
                                 <li><a href="running-on-yarn.html">YARN</a></li>
                                 <li><a href="running-on-kubernetes.html">Kubernetes</a></li>
                             </ul>
                         </li>

                         <li class="dropdown">
                             <a href="api.html" class="dropdown-toggle" data-toggle="dropdown">More<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="configuration.html">Configuration</a></li>
                                 <li><a href="monitoring.html">Monitoring</a></li>
                                 <li><a href="tuning.html">Tuning Guide</a></li>
                                 <li><a href="job-scheduling.html">Job Scheduling</a></li>
                                 <li><a href="security.html">Security</a></li>
                                 <li><a href="hardware-provisioning.html">Hardware Provisioning</a></li>
                                 <li class="divider"></li>
                                 <li><a href="building-spark.html">Building Spark</a></li>
                                 <li><a href="https://spark.apache.org/contributing.html">Contributing to Spark</a></li>
                                 <li><a href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a></li>
                             </ul>
                         </li>
                     </ul>
                     <!--<p class="navbar-text pull-right"><span class="version-text">v2.4.5</span></p>-->
                 </div>
             </div>
         </div>

         <div class="container-wrapper">


                     <div class="left-menu-wrapper">
     <div class="left-menu">
         <h3><a href="sql-programming-guide.html">Spark SQL Guide</a></h3>

 <ul>

     <li>
         <a href="sql-getting-started.html">

                 Getting Started

         </a>
     </li>


     <li>
         <a href="sql-data-sources.html">

                 Data Sources

         </a>
     </li>


 <ul>

     <li>
         <a href="sql-data-sources-load-save-functions.html">

                 Generic Load/Save Functions

         </a>
     </li>


     <li>
         <a href="sql-data-sources-parquet.html">

                 Parquet Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-orc.html">

                 <b>ORC Files</b>

         </a>
     </li>


     <li>
         <a href="sql-data-sources-json.html">

                 JSON Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-hive-tables.html">

                 Hive Tables

         </a>
     </li>


     <li>
         <a href="sql-data-sources-jdbc.html">

                 JDBC To Other Databases

         </a>
     </li>


     <li>
         <a href="sql-data-sources-avro.html">

                 Avro Files

         </a>
     </li>


     <li>
         <a href="sql-data-sources-troubleshooting.html">

                 Troubleshooting

         </a>
     </li>


 </ul>


     <li>
         <a href="sql-performance-tuning.html">

                 Performance Tuning

         </a>
     </li>


     <li>
         <a href="sql-distributed-sql-engine.html">

                 Distributed SQL Engine

         </a>
     </li>


     <li>
         <a href="sql-pyspark-pandas-with-arrow.html">

                 PySpark Usage Guide for Pandas with Apache Arrow

         </a>
     </li>


     <li>
         <a href="sql-migration-guide.html">

                 Migration Guide

         </a>
     </li>


     <li>
         <a href="sql-reference.html">

                 Reference

         </a>
     </li>


 </ul>

     </div>
 </div>

                 <input id="nav-trigger" class="nav-trigger" checked type="checkbox">
                 <label for="nav-trigger"></label>
                 <div class="content-with-sidebar" id="content">

                         <h1 class="title">ORC Files</h1>


                     <p>Since Spark 2.3, Spark supports a vectorized ORC reader with a new ORC file format for ORC files.
 To do that, the following configurations are newly added. The vectorized reader is used for the
 native ORC tables (e.g., the ones created using the clause <code>USING ORC</code>) when <code>spark.sql.orc.impl</code>
 is set to <code>native</code> and <code>spark.sql.orc.enableVectorizedReader</code> is set to <code>true</code>. For the Hive ORC
 serde tables (e.g., the ones created using the clause <code>USING HIVE OPTIONS (fileFormat 'ORC')</code>),
 the vectorized reader is used when <code>spark.sql.hive.convertMetastoreOrc</code> is also set to <code>true</code>.</p>

 <table class="table">
   <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th></tr>
   <tr>
     <td><code>spark.sql.orc.impl</code></td>
     <td><code>native</code></td>
     <td>The name of ORC implementation. It can be one of <code>native</code> and <code>hive</code>. <code>native</code> means the native ORC support that is built on Apache ORC 1.4. `hive` means the ORC library in Hive 1.2.1.</td>
   </tr>
   <tr>
     <td><code>spark.sql.orc.enableVectorizedReader</code></td>
     <td><code>true</code></td>
     <td>Enables vectorized orc decoding in <code>native</code> implementation. If <code>false</code>, a new non-vectorized ORC reader is used in <code>native</code> implementation. For <code>hive</code> implementation, this is ignored.</td>
   </tr>
 </table>


                 </div>

              <!-- /container -->
         </div>

         <script src="js/vendor/jquery-1.12.4.min.js"></script>
         <script src="js/vendor/bootstrap.min.js"></script>
         <script src="js/vendor/anchor.min.js"></script>
         <script src="js/main.js"></script>

         <!-- MathJax Section -->
         <script type="text/x-mathjax-config">
             MathJax.Hub.Config({
                 TeX: { equationNumbers: { autoNumber: "AMS" } }
             });
         </script>
         <script>
             // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
             // We could use "//cdn.mathjax...", but that won't support "file://".
             (function(d, script) {
                 script = d.createElement('script');
                 script.type = 'text/javascript';
                 script.async = true;
                 script.onload = function(){
                     MathJax.Hub.Config({
                         tex2jax: {
                             inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
                             displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
                             processEscapes: true,
                             skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
                         }
                     });
                 };
                 script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
                     'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
                     '?config=TeX-AMS-MML_HTMLorMML';
                 d.getElementsByTagName('head')[0].appendChild(script);
             }(document));
         </script>
     </body>
 </html>

	<!DOCTYPE html>
	<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
	<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
	<!--[if IE 8]> <html class="no-js lt-ie9"> <![endif]-->
	<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
	<head>
	<meta charset="utf-8">
	<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
	<title>ORC Files - Spark 2.4.5 Documentation</title>




	<link rel="stylesheet" href="css/bootstrap.min.css">
	<style>
	body {
	padding-top: 60px;
	padding-bottom: 40px;
	}
	</style>
	<meta name="viewport" content="width=device-width">
	<link rel="stylesheet" href="css/bootstrap-responsive.min.css">
	<link rel="stylesheet" href="css/main.css">

	<script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>

	<link rel="stylesheet" href="css/pygments-default.css">


	<!-- Google analytics script -->
	<script type="text/javascript">
	var _gaq = _gaq \|\| [];
	_gaq.push(['_setAccount', 'UA-32518208-2']);
	_gaq.push(['_trackPageview']);

	(function() {
	var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
	ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
	var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
	})();
	</script>


	</head>
	<body>
	<!--[if lt IE 7]>
	<p class="chromeframe">You are using an outdated browser. <a href="https://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p>
	<![endif]-->

	<!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->

	<div class="navbar navbar-fixed-top" id="topbar">
	<div class="navbar-inner">
	<div class="container">
	<div class="brand"><a href="index.html">
	<img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">2.4.5</span>
	</div>
	<ul class="nav">
	<!--TODO(andyk): Add class="active" attribute to li some how.-->
	<li><a href="index.html">Overview</a></li>

	<li class="dropdown">
	<a href="#" class="dropdown-toggle" data-toggle="dropdown">Programming Guides<b class="caret"></b></a>
	<ul class="dropdown-menu">
	<li><a href="quick-start.html">Quick Start</a></li>
	<li><a href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a></li>
	<li><a href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a></li>
	<li><a href="structured-streaming-programming-guide.html">Structured Streaming</a></li>
	<li><a href="streaming-programming-guide.html">Spark Streaming (DStreams)</a></li>
	<li><a href="ml-guide.html">MLlib (Machine Learning)</a></li>
	<li><a href="graphx-programming-guide.html">GraphX (Graph Processing)</a></li>
	<li><a href="sparkr.html">SparkR (R on Spark)</a></li>
	</ul>
	</li>

	<li class="dropdown">
	<a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a>
	<ul class="dropdown-menu">
	<li><a href="api/scala/index.html#org.apache.spark.package">Scala</a></li>
	<li><a href="api/java/index.html">Java</a></li>
	<li><a href="api/python/index.html">Python</a></li>
	<li><a href="api/R/index.html">R</a></li>
	<li><a href="api/sql/index.html">SQL, Built-in Functions</a></li>
	</ul>
	</li>

	<li class="dropdown">
	<a href="#" class="dropdown-toggle" data-toggle="dropdown">Deploying<b class="caret"></b></a>
	<ul class="dropdown-menu">
	<li><a href="cluster-overview.html">Overview</a></li>
	<li><a href="submitting-applications.html">Submitting Applications</a></li>
	<li class="divider"></li>
	<li><a href="spark-standalone.html">Spark Standalone</a></li>
	<li><a href="running-on-mesos.html">Mesos</a></li>
	<li><a href="running-on-yarn.html">YARN</a></li>
	<li><a href="running-on-kubernetes.html">Kubernetes</a></li>
	</ul>
	</li>

	<li class="dropdown">
	<a href="api.html" class="dropdown-toggle" data-toggle="dropdown">More<b class="caret"></b></a>
	<ul class="dropdown-menu">
	<li><a href="configuration.html">Configuration</a></li>
	<li><a href="monitoring.html">Monitoring</a></li>
	<li><a href="tuning.html">Tuning Guide</a></li>
	<li><a href="job-scheduling.html">Job Scheduling</a></li>
	<li><a href="security.html">Security</a></li>
	<li><a href="hardware-provisioning.html">Hardware Provisioning</a></li>
	<li class="divider"></li>
	<li><a href="building-spark.html">Building Spark</a></li>
	<li><a href="https://spark.apache.org/contributing.html">Contributing to Spark</a></li>
	<li><a href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a></li>
	</ul>
	</li>
	</ul>
	<!--<p class="navbar-text pull-right"><span class="version-text">v2.4.5</span></p>-->
	</div>
	</div>
	</div>

	<div class="container-wrapper">



	<div class="left-menu-wrapper">
	<div class="left-menu">
	<h3><a href="sql-programming-guide.html">Spark SQL Guide</a></h3>

	<ul>

	<li>
	<a href="sql-getting-started.html">

	Getting Started

	</a>
	</li>



	<li>
	<a href="sql-data-sources.html">

	Data Sources

	</a>
	</li>



	<ul>

	<li>
	<a href="sql-data-sources-load-save-functions.html">

	Generic Load/Save Functions

	</a>
	</li>



	<li>
	<a href="sql-data-sources-parquet.html">

	Parquet Files

	</a>
	</li>



	<li>
	<a href="sql-data-sources-orc.html">

	<b>ORC Files</b>

	</a>
	</li>



	<li>
	<a href="sql-data-sources-json.html">

	JSON Files

	</a>
	</li>



	<li>
	<a href="sql-data-sources-hive-tables.html">

	Hive Tables

	</a>
	</li>



	<li>
	<a href="sql-data-sources-jdbc.html">

	JDBC To Other Databases

	</a>
	</li>



	<li>
	<a href="sql-data-sources-avro.html">

	Avro Files

	</a>
	</li>



	<li>
	<a href="sql-data-sources-troubleshooting.html">

	Troubleshooting

	</a>
	</li>



	</ul>



	<li>
	<a href="sql-performance-tuning.html">

	Performance Tuning

	</a>
	</li>



	<li>
	<a href="sql-distributed-sql-engine.html">

	Distributed SQL Engine

	</a>
	</li>



	<li>
	<a href="sql-pyspark-pandas-with-arrow.html">

	PySpark Usage Guide for Pandas with Apache Arrow

	</a>
	</li>



	<li>
	<a href="sql-migration-guide.html">

	Migration Guide

	</a>
	</li>



	<li>
	<a href="sql-reference.html">

	Reference

	</a>
	</li>



	</ul>

	</div>
	</div>

	<input id="nav-trigger" class="nav-trigger" checked type="checkbox">
	<label for="nav-trigger"></label>
	<div class="content-with-sidebar" id="content">

	<h1 class="title">ORC Files</h1>


	<p>Since Spark 2.3, Spark supports a vectorized ORC reader with a new ORC file format for ORC files.
	To do that, the following configurations are newly added. The vectorized reader is used for the
	native ORC tables (e.g., the ones created using the clause <code>USING ORC</code>) when <code>spark.sql.orc.impl</code>
	is set to <code>native</code> and <code>spark.sql.orc.enableVectorizedReader</code> is set to <code>true</code>. For the Hive ORC
	serde tables (e.g., the ones created using the clause <code>USING HIVE OPTIONS (fileFormat 'ORC')</code>),
	the vectorized reader is used when <code>spark.sql.hive.convertMetastoreOrc</code> is also set to <code>true</code>.</p>

	<table class="table">
	<tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th></tr>
	<tr>
	<td><code>spark.sql.orc.impl</code></td>
	<td><code>native</code></td>
	<td>The name of ORC implementation. It can be one of <code>native</code> and <code>hive</code>. <code>native</code> means the native ORC support that is built on Apache ORC 1.4. `hive` means the ORC library in Hive 1.2.1.</td>
	</tr>
	<tr>
	<td><code>spark.sql.orc.enableVectorizedReader</code></td>
	<td><code>true</code></td>
	<td>Enables vectorized orc decoding in <code>native</code> implementation. If <code>false</code>, a new non-vectorized ORC reader is used in <code>native</code> implementation. For <code>hive</code> implementation, this is ignored.</td>
	</tr>
	</table>


	</div>

	<!-- /container -->
	</div>

	<script src="js/vendor/jquery-1.12.4.min.js"></script>
	<script src="js/vendor/bootstrap.min.js"></script>
	<script src="js/vendor/anchor.min.js"></script>
	<script src="js/main.js"></script>

	<!-- MathJax Section -->
	<script type="text/x-mathjax-config">
	MathJax.Hub.Config({
	TeX: { equationNumbers: { autoNumber: "AMS" } }
	});
	</script>
	<script>
	// Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
	// We could use "//cdn.mathjax...", but that won't support "file://".
	(function(d, script) {
	script = d.createElement('script');
	script.type = 'text/javascript';
	script.async = true;
	script.onload = function(){
	MathJax.Hub.Config({
	tex2jax: {
	inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
	displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
	processEscapes: true,
	skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
	}
	});
	};
	script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
	'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
	'?config=TeX-AMS-MML_HTMLorMML';
	d.getElementsByTagName('head')[0].appendChild(script);
	}(document));
	</script>
	</body>
	</html>