blob: 304217bee55bc33f400236f14b4ba74da36b67a5 [file] [log] [blame]
<!DOCTYPE html>
<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
<!--[if IE 8]> <html class="no-js lt-ie9"> <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<title>INSERT OVERWRITE DIRECTORY - Spark 3.3.4 Documentation</title>
<link rel="stylesheet" href="css/bootstrap.min.css">
<style>
body {
padding-top: 60px;
padding-bottom: 40px;
}
</style>
<meta name="viewport" content="width=device-width">
<link rel="stylesheet" href="css/main.css">
<script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>
<link rel="stylesheet" href="css/pygments-default.css">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css" />
<link rel="stylesheet" href="css/docsearch.css">
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '40']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body>
<!--[if lt IE 7]>
<p class="chromeframe">You are using an outdated browser. <a href="https://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p>
<![endif]-->
<!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->
<nav class="navbar fixed-top navbar-expand-md navbar-light bg-light" id="topbar">
<div class="container">
<div class="navbar-header">
<div class="navbar-brand"><a href="index.html">
<img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">3.3.4</span>
</div>
</div>
<button class="navbar-toggler" type="button" data-toggle="collapse"
data-target="#navbarCollapse" aria-controls="navbarCollapse"
aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarCollapse">
<ul class="navbar-nav">
<!--TODO(andyk): Add class="active" attribute to li some how.-->
<li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li>
<li class="nav-item dropdown">
<a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a>
<div class="dropdown-menu" aria-labelledby="navbarQuickStart">
<a class="dropdown-item" href="quick-start.html">Quick Start</a>
<a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a>
<a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a>
<a class="dropdown-item" href="structured-streaming-programming-guide.html">Structured Streaming</a>
<a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a>
<a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a>
<a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a>
<a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a>
<a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a>
</div>
</li>
<li class="nav-item dropdown">
<a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a>
<div class="dropdown-menu" aria-labelledby="navbarAPIDocs">
<a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a>
<a class="dropdown-item" href="api/java/index.html">Java</a>
<a class="dropdown-item" href="api/python/index.html">Python</a>
<a class="dropdown-item" href="api/R/index.html">R</a>
<a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a>
</div>
</li>
<li class="nav-item dropdown">
<a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a>
<div class="dropdown-menu" aria-labelledby="navbarDeploying">
<a class="dropdown-item" href="cluster-overview.html">Overview</a>
<a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a>
<div class="dropdown-divider"></div>
<a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a>
<a class="dropdown-item" href="running-on-mesos.html">Mesos</a>
<a class="dropdown-item" href="running-on-yarn.html">YARN</a>
<a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a>
</div>
</li>
<li class="nav-item dropdown">
<a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
<div class="dropdown-menu" aria-labelledby="navbarMore">
<a class="dropdown-item" href="configuration.html">Configuration</a>
<a class="dropdown-item" href="monitoring.html">Monitoring</a>
<a class="dropdown-item" href="tuning.html">Tuning Guide</a>
<a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a>
<a class="dropdown-item" href="security.html">Security</a>
<a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a>
<a class="dropdown-item" href="migration-guide.html">Migration Guide</a>
<div class="dropdown-divider"></div>
<a class="dropdown-item" href="building-spark.html">Building Spark</a>
<a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a>
<a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a>
</div>
</li>
<li class="nav-item">
<input type="text" id="docsearch-input" placeholder="Search the docs…">
</li>
</ul>
<!--<span class="navbar-text navbar-right"><span class="version-text">v3.3.4</span></span>-->
</div>
</div>
</nav>
<div class="container-wrapper">
<div class="left-menu-wrapper">
<div class="left-menu">
<h3><a href="sql-programming-guide.html">Spark SQL Guide</a></h3>
<ul>
<li>
<a href="sql-getting-started.html">
Getting Started
</a>
</li>
<li>
<a href="sql-data-sources.html">
Data Sources
</a>
</li>
<li>
<a href="sql-performance-tuning.html">
Performance Tuning
</a>
</li>
<li>
<a href="sql-distributed-sql-engine.html">
Distributed SQL Engine
</a>
</li>
<li>
<a href="sql-pyspark-pandas-with-arrow.html">
PySpark Usage Guide for Pandas with Apache Arrow
</a>
</li>
<li>
<a href="sql-migration-old.html">
Migration Guide
</a>
</li>
<li>
<a href="sql-ref.html">
SQL Reference
</a>
</li>
<ul>
<li>
<a href="sql-ref-ansi-compliance.html">
ANSI Compliance
</a>
</li>
<li>
<a href="sql-ref-datatypes.html">
Data Types
</a>
</li>
<li>
<a href="sql-ref-datetime-pattern.html">
Datetime Pattern
</a>
</li>
<li>
<a href="sql-ref-number-pattern.html">
Number Pattern
</a>
</li>
<li>
<a href="sql-ref-functions.html">
Functions
</a>
</li>
<li>
<a href="sql-ref-identifier.html">
Identifiers
</a>
</li>
<li>
<a href="sql-ref-literals.html">
Literals
</a>
</li>
<li>
<a href="sql-ref-null-semantics.html">
Null Semantics
</a>
</li>
<li>
<a href="sql-ref-syntax.html">
SQL Syntax
</a>
</li>
<ul>
<li>
<a href="sql-ref-syntax.html#ddl-statements">
Data Definition Statements
</a>
</li>
<li>
<a href="sql-ref-syntax.html#dml-statements">
Data Manipulation Statements
</a>
</li>
<li>
<a href="sql-ref-syntax.html#data-retrieval-statements">
Data Retrieval(Queries)
</a>
</li>
<li>
<a href="sql-ref-syntax.html#auxiliary-statements">
Auxiliary Statements
</a>
</li>
</ul>
</ul>
</ul>
</div>
</div>
<input id="nav-trigger" class="nav-trigger" checked type="checkbox">
<label for="nav-trigger"></label>
<div class="content-with-sidebar mr-3" id="content">
<h1 class="title">INSERT OVERWRITE DIRECTORY</h1>
<h3 id="description">Description</h3>
<p>The <code class="language-plaintext highlighter-rouge">INSERT OVERWRITE DIRECTORY</code> statement overwrites the existing data in the directory with the new values using either spark file format or Hive Serde.
Hive support must be enabled to use Hive Serde. The inserted rows can be specified by value expressions or result from a query.</p>
<h3 id="syntax">Syntax</h3>
<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">INSERT</span> <span class="n">OVERWRITE</span> <span class="p">[</span> <span class="k">LOCAL</span> <span class="p">]</span> <span class="n">DIRECTORY</span> <span class="p">[</span> <span class="n">directory_path</span> <span class="p">]</span>
<span class="err">{</span> <span class="n">spark_format</span> <span class="o">|</span> <span class="n">hive_format</span> <span class="err">}</span>
<span class="err">{</span> <span class="k">VALUES</span> <span class="p">(</span> <span class="err">{</span> <span class="n">value</span> <span class="o">|</span> <span class="k">NULL</span> <span class="err">}</span> <span class="p">[</span> <span class="p">,</span> <span class="p">...</span> <span class="p">]</span> <span class="p">)</span> <span class="p">[</span> <span class="p">,</span> <span class="p">(</span> <span class="p">...</span> <span class="p">)</span> <span class="p">]</span> <span class="o">|</span> <span class="n">query</span> <span class="err">}</span>
</code></pre></div></div>
<p>While <code class="language-plaintext highlighter-rouge">spark_format</code> is defined as</p>
<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">USING</span> <span class="n">file_format</span> <span class="p">[</span> <span class="k">OPTIONS</span> <span class="p">(</span> <span class="k">key</span> <span class="o">=</span> <span class="n">val</span> <span class="p">[</span> <span class="p">,</span> <span class="p">...</span> <span class="p">]</span> <span class="p">)</span> <span class="p">]</span>
</code></pre></div></div>
<p><code class="language-plaintext highlighter-rouge">hive_format</code> is defined as</p>
<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="p">[</span> <span class="k">ROW</span> <span class="n">FORMAT</span> <span class="n">row_format</span> <span class="p">]</span> <span class="p">[</span> <span class="n">STORED</span> <span class="k">AS</span> <span class="n">hive_serde</span> <span class="p">]</span>
</code></pre></div></div>
<h3 id="parameters">Parameters</h3>
<ul>
<li>
<p><strong>directory_path</strong></p>
<p>Specifies the destination directory. The <code class="language-plaintext highlighter-rouge">LOCAL</code> keyword is used to specify that the directory is on the local file system.
In spark file format, it can also be specified in <code class="language-plaintext highlighter-rouge">OPTIONS</code> using <code class="language-plaintext highlighter-rouge">path</code>, but <code class="language-plaintext highlighter-rouge">directory_path</code> and <code class="language-plaintext highlighter-rouge">path</code> option can not be both specified.</p>
</li>
<li>
<p><strong>file_format</strong></p>
<p>Specifies the file format to use for the insert. Valid options are <code class="language-plaintext highlighter-rouge">TEXT</code>, <code class="language-plaintext highlighter-rouge">CSV</code>, <code class="language-plaintext highlighter-rouge">JSON</code>, <code class="language-plaintext highlighter-rouge">JDBC</code>, <code class="language-plaintext highlighter-rouge">PARQUET</code>, <code class="language-plaintext highlighter-rouge">ORC</code>, <code class="language-plaintext highlighter-rouge">HIVE</code>, <code class="language-plaintext highlighter-rouge">LIBSVM</code>, or a fully qualified class name of a custom implementation of <code class="language-plaintext highlighter-rouge">org.apache.spark.sql.execution.datasources.FileFormat</code>.</p>
</li>
<li>
<p><strong>OPTIONS ( key = val [ , &#8230; ] )</strong></p>
<p>Specifies one or more options for the writing of the file format.</p>
</li>
<li>
<p><strong>hive_format</strong></p>
<p>Specifies the file format to use for the insert. Both <code class="language-plaintext highlighter-rouge">row_format</code> and <code class="language-plaintext highlighter-rouge">hive_serde</code> are optional. <code class="language-plaintext highlighter-rouge">ROW FORMAT SERDE</code> can only be used with <code class="language-plaintext highlighter-rouge">TEXTFILE</code>, <code class="language-plaintext highlighter-rouge">SEQUENCEFILE</code>, or <code class="language-plaintext highlighter-rouge">RCFILE</code>, while <code class="language-plaintext highlighter-rouge">ROW FORMAT DELIMITED</code> can only be used with <code class="language-plaintext highlighter-rouge">TEXTFILE</code>. If both are not defined, spark uses <code class="language-plaintext highlighter-rouge">TEXTFILE</code>.</p>
</li>
<li>
<p><strong>row_format</strong></p>
<p>Specifies the row format for this insert. Valid options are <code class="language-plaintext highlighter-rouge">SERDE</code> clause and <code class="language-plaintext highlighter-rouge">DELIMITED</code> clause. <code class="language-plaintext highlighter-rouge">SERDE</code> clause can be used to specify a custom <code class="language-plaintext highlighter-rouge">SerDe</code> for this insert. Alternatively, <code class="language-plaintext highlighter-rouge">DELIMITED</code> clause can be used to specify the native <code class="language-plaintext highlighter-rouge">SerDe</code> and state the delimiter, escape character, null character, and so on.</p>
</li>
<li>
<p><strong>hive_serde</strong></p>
<p>Specifies the file format for this insert. Valid options are <code class="language-plaintext highlighter-rouge">TEXTFILE</code>, <code class="language-plaintext highlighter-rouge">SEQUENCEFILE</code>, <code class="language-plaintext highlighter-rouge">RCFILE</code>, <code class="language-plaintext highlighter-rouge">ORC</code>, <code class="language-plaintext highlighter-rouge">PARQUET</code>, and <code class="language-plaintext highlighter-rouge">AVRO</code>. You can also specify your own input and output format using <code class="language-plaintext highlighter-rouge">INPUTFORMAT</code> and <code class="language-plaintext highlighter-rouge">OUTPUTFORMAT</code>.</p>
</li>
<li>
<p><strong>VALUES ( { value <code class="language-plaintext highlighter-rouge">|</code> NULL } [ , &#8230; ] ) [ , ( &#8230; ) ]</strong></p>
<p>Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.</p>
</li>
<li>
<p><strong>query</strong></p>
<p>A query that produces the rows to be inserted. It can be in one of following formats:</p>
<ul>
<li>a <a href="sql-ref-syntax-qry-select.html">SELECT</a> statement</li>
<li>a <a href="sql-ref-syntax-qry-select-inline-table.html">Inline Table</a> statement</li>
<li>a <code class="language-plaintext highlighter-rouge">FROM</code> statement</li>
</ul>
</li>
</ul>
<h3 id="examples">Examples</h3>
<h4 id="spark-format">Spark format</h4>
<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">INSERT</span> <span class="n">OVERWRITE</span> <span class="n">DIRECTORY</span> <span class="s1">'/tmp/destination'</span>
<span class="k">USING</span> <span class="n">parquet</span>
<span class="k">OPTIONS</span> <span class="p">(</span><span class="n">col1</span> <span class="mi">1</span><span class="p">,</span> <span class="n">col2</span> <span class="mi">2</span><span class="p">,</span> <span class="n">col3</span> <span class="s1">'test'</span><span class="p">)</span>
<span class="k">SELECT</span> <span class="o">*</span> <span class="k">FROM</span> <span class="n">test_table</span><span class="p">;</span>
<span class="k">INSERT</span> <span class="n">OVERWRITE</span> <span class="n">DIRECTORY</span>
<span class="k">USING</span> <span class="n">parquet</span>
<span class="k">OPTIONS</span> <span class="p">(</span><span class="s1">'path'</span> <span class="s1">'/tmp/destination'</span><span class="p">,</span> <span class="n">col1</span> <span class="mi">1</span><span class="p">,</span> <span class="n">col2</span> <span class="mi">2</span><span class="p">,</span> <span class="n">col3</span> <span class="s1">'test'</span><span class="p">)</span>
<span class="k">SELECT</span> <span class="o">*</span> <span class="k">FROM</span> <span class="n">test_table</span><span class="p">;</span>
</code></pre></div></div>
<h4 id="hive-format">Hive format</h4>
<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">INSERT</span> <span class="n">OVERWRITE</span> <span class="k">LOCAL</span> <span class="n">DIRECTORY</span> <span class="s1">'/tmp/destination'</span>
<span class="n">STORED</span> <span class="k">AS</span> <span class="n">orc</span>
<span class="k">SELECT</span> <span class="o">*</span> <span class="k">FROM</span> <span class="n">test_table</span><span class="p">;</span>
<span class="k">INSERT</span> <span class="n">OVERWRITE</span> <span class="k">LOCAL</span> <span class="n">DIRECTORY</span> <span class="s1">'/tmp/destination'</span>
<span class="k">ROW</span> <span class="n">FORMAT</span> <span class="n">DELIMITED</span> <span class="n">FIELDS</span> <span class="n">TERMINATED</span> <span class="k">BY</span> <span class="s1">','</span>
<span class="k">SELECT</span> <span class="o">*</span> <span class="k">FROM</span> <span class="n">test_table</span><span class="p">;</span>
</code></pre></div></div>
<h3 id="related-statements">Related Statements</h3>
<ul>
<li><a href="sql-ref-syntax-dml-insert-table.html">INSERT TABLE statement</a></li>
</ul>
</div>
<!-- /container -->
</div>
<script src="js/vendor/jquery-3.5.1.min.js"></script>
<script src="js/vendor/bootstrap.bundle.min.js"></script>
<script src="js/vendor/anchor.min.js"></script>
<script src="js/main.js"></script>
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js"></script>
<script type="text/javascript">
// DocSearch is entirely free and automated. DocSearch is built in two parts:
// 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link
// in your website and extract content from every page it traverses. It then pushes this
// content to an Algolia index.
// 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index
// to your search input and display its results in a dropdown UI. If you want to find more
// details on how works DocSearch, check the docs of DocSearch.
docsearch({
apiKey: 'd62f962a82bc9abb53471cb7b89da35e',
appId: 'RAI69RXRSK',
indexName: 'apache_spark',
inputSelector: '#docsearch-input',
enhancedSearchInput: true,
algoliaOptions: {
'facetFilters': ["version:3.3.4"]
},
debug: false // Set debug to true if you want to inspect the dropdown
});
</script>
<!-- MathJax Section -->
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>
<script>
// Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
// We could use "//cdn.mathjax...", but that won't support "file://".
(function(d, script) {
script = d.createElement('script');
script.type = 'text/javascript';
script.async = true;
script.onload = function(){
MathJax.Hub.Config({
tex2jax: {
inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
processEscapes: true,
skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
}
});
};
script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
'?config=TeX-AMS-MML_HTMLorMML';
d.getElementsByTagName('head')[0].appendChild(script);
}(document));
</script>
</body>
</html>