blob: e7fd0cb8bc8f90925a0eb712bf3d61ab16f2d354 [file] [log] [blame]
<!DOCTYPE HTML>
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc -->
<title>SparkWriteConf</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<link rel="stylesheet" type="text/css" href="../../../../jquery/jquery-ui.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip/dist/jszip.min.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils.min.js"></script>
<!--[if IE]>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils-ie.min.js"></script>
<![endif]-->
<script type="text/javascript" src="../../../../jquery/jquery-3.5.1.js"></script>
<script type="text/javascript" src="../../../../jquery/jquery-ui.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="SparkWriteConf";
}
}
catch(err) {
}
//-->
var data = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
var pathtoroot = "../../../../";
var useModuleDirectories = true;
loadScripts(document, 'script');</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<header role="banner">
<nav role="navigation">
<div class="fixedNav">
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a id="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<ul class="navListSearch">
<li><label for="search">SEARCH:</label>
<input type="text" id="search" value="search" disabled="disabled">
<input type="reset" id="reset" value="reset" disabled="disabled">
</li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
</div>
<div class="navPadding">&nbsp;</div>
<script type="text/javascript"><!--
$('.navPadding').css('padding-top', $('.fixedNav').css("height"));
//-->
</script>
</nav>
</header>
<!-- ======== START OF CLASS DATA ======== -->
<main role="main">
<div class="header">
<div class="subTitle"><span class="packageLabelInType">Package</span>&nbsp;<a href="package-summary.html">org.apache.iceberg.spark</a></div>
<h2 title="Class SparkWriteConf" class="title">Class SparkWriteConf</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>org.apache.iceberg.spark.SparkWriteConf</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<hr>
<pre>public class <span class="typeNameLabel">SparkWriteConf</span>
extends java.lang.Object</pre>
<div class="block">A class for common Iceberg configs for Spark writes.
<p>
If a config is set at multiple levels, the following order of precedence is used (top to bottom):
<ol>
<li>Write options</li>
<li>Session configuration</li>
<li>Table metadata</li>
</ol>
The most specific value is set in write options and takes precedence over all other configs.
If no write option is provided, this class checks the session configuration for any overrides.
If no applicable value is found in the session configuration, this class uses the table metadata.
<p>
Note this class is NOT meant to be serialized and sent to executors.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="constructor.summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="memberSummary">
<caption><span>Constructors</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Constructor</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tr class="altColor">
<th class="colConstructorName" scope="row"><code><span class="memberNameLink"><a href="#%3Cinit%3E(org.apache.spark.sql.SparkSession,org.apache.iceberg.Table,java.util.Map)">SparkWriteConf</a></span>&#8203;(org.apache.spark.sql.SparkSession&nbsp;spark,
<a href="../Table.html" title="interface in org.apache.iceberg">Table</a>&nbsp;table,
java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;writeOptions)</code></th>
<td class="colLast">&nbsp;</td>
</tr>
</table>
</li>
</ul>
</section>
<!-- ========== METHOD SUMMARY =========== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colSecond" scope="col">Method</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#checkNullability()">checkNullability</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#checkOrdering()">checkOrdering</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code><a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#copyOnWriteDeleteDistributionMode()">copyOnWriteDeleteDistributionMode</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code><a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#copyOnWriteMergeDistributionMode()">copyOnWriteMergeDistributionMode</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code><a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#copyOnWriteUpdateDistributionMode()">copyOnWriteUpdateDistributionMode</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code><a href="../FileFormat.html" title="enum in org.apache.iceberg">FileFormat</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#dataFileFormat()">dataFileFormat</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code><a href="../FileFormat.html" title="enum in org.apache.iceberg">FileFormat</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#deleteFileFormat()">deleteFileFormat</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i7" class="rowColor">
<td class="colFirst"><code><a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#distributionMode()">distributionMode</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i8" class="altColor">
<td class="colFirst"><code>java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#extraSnapshotMetadata()">extraSnapshotMetadata</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i9" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#fanoutWriterEnabled()">fanoutWriterEnabled</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i10" class="altColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#handleTimestampWithoutZone()">handleTimestampWithoutZone</a></span>()</code></th>
<td class="colLast">
<div class="block">Enables writing a timestamp with time zone as a timestamp without time zone.</div>
</td>
</tr>
<tr id="i11" class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#overwriteMode()">overwriteMode</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i12" class="altColor">
<td class="colFirst"><code><a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#positionDeleteDistributionMode()">positionDeleteDistributionMode</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i13" class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#rewrittenFileSetId()">rewrittenFileSetId</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i14" class="altColor">
<td class="colFirst"><code>long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#targetDataFileSize()">targetDataFileSize</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i15" class="rowColor">
<td class="colFirst"><code>long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#targetDeleteFileSize()">targetDeleteFileSize</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i16" class="altColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#useTableDistributionAndOrdering()">useTableDistributionAndOrdering</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i17" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#wapEnabled()">wapEnabled</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i18" class="altColor">
<td class="colFirst"><code>java.lang.String</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#wapId()">wapId</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a id="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="constructor.detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a id="&lt;init&gt;(org.apache.spark.sql.SparkSession,org.apache.iceberg.Table,java.util.Map)">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>SparkWriteConf</h4>
<pre>public&nbsp;SparkWriteConf&#8203;(org.apache.spark.sql.SparkSession&nbsp;spark,
<a href="../Table.html" title="interface in org.apache.iceberg">Table</a>&nbsp;table,
java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;writeOptions)</pre>
</li>
</ul>
</li>
</ul>
</section>
<!-- ============ METHOD DETAIL ========== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a id="checkNullability()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>checkNullability</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;checkNullability()</pre>
</li>
</ul>
<a id="checkOrdering()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>checkOrdering</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;checkOrdering()</pre>
</li>
</ul>
<a id="handleTimestampWithoutZone()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>handleTimestampWithoutZone</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;handleTimestampWithoutZone()</pre>
<div class="block">Enables writing a timestamp with time zone as a timestamp without time zone.
<p>
Generally, this is not safe as a timestamp without time zone is supposed to represent the wall-clock time,
i.e. no matter the reader/writer timezone 3PM should always be read as 3PM,
but a timestamp with time zone represents instant semantics, i.e. the timestamp
is adjusted so that the corresponding time in the reader timezone is displayed.
<p>
When set to false (default), an exception must be thrown if the table contains a timestamp without time zone.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>boolean indicating if writing timestamps without timezone is allowed</dd>
</dl>
</li>
</ul>
<a id="overwriteMode()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>overwriteMode</h4>
<pre class="methodSignature">public&nbsp;java.lang.String&nbsp;overwriteMode()</pre>
</li>
</ul>
<a id="wapEnabled()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>wapEnabled</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;wapEnabled()</pre>
</li>
</ul>
<a id="wapId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>wapId</h4>
<pre class="methodSignature">public&nbsp;java.lang.String&nbsp;wapId()</pre>
</li>
</ul>
<a id="dataFileFormat()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>dataFileFormat</h4>
<pre class="methodSignature">public&nbsp;<a href="../FileFormat.html" title="enum in org.apache.iceberg">FileFormat</a>&nbsp;dataFileFormat()</pre>
</li>
</ul>
<a id="targetDataFileSize()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>targetDataFileSize</h4>
<pre class="methodSignature">public&nbsp;long&nbsp;targetDataFileSize()</pre>
</li>
</ul>
<a id="fanoutWriterEnabled()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>fanoutWriterEnabled</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;fanoutWriterEnabled()</pre>
</li>
</ul>
<a id="deleteFileFormat()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>deleteFileFormat</h4>
<pre class="methodSignature">public&nbsp;<a href="../FileFormat.html" title="enum in org.apache.iceberg">FileFormat</a>&nbsp;deleteFileFormat()</pre>
</li>
</ul>
<a id="targetDeleteFileSize()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>targetDeleteFileSize</h4>
<pre class="methodSignature">public&nbsp;long&nbsp;targetDeleteFileSize()</pre>
</li>
</ul>
<a id="extraSnapshotMetadata()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>extraSnapshotMetadata</h4>
<pre class="methodSignature">public&nbsp;java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;extraSnapshotMetadata()</pre>
</li>
</ul>
<a id="rewrittenFileSetId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>rewrittenFileSetId</h4>
<pre class="methodSignature">public&nbsp;java.lang.String&nbsp;rewrittenFileSetId()</pre>
</li>
</ul>
<a id="distributionMode()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>distributionMode</h4>
<pre class="methodSignature">public&nbsp;<a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a>&nbsp;distributionMode()</pre>
</li>
</ul>
<a id="copyOnWriteDeleteDistributionMode()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>copyOnWriteDeleteDistributionMode</h4>
<pre class="methodSignature">public&nbsp;<a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a>&nbsp;copyOnWriteDeleteDistributionMode()</pre>
</li>
</ul>
<a id="copyOnWriteUpdateDistributionMode()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>copyOnWriteUpdateDistributionMode</h4>
<pre class="methodSignature">public&nbsp;<a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a>&nbsp;copyOnWriteUpdateDistributionMode()</pre>
</li>
</ul>
<a id="copyOnWriteMergeDistributionMode()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>copyOnWriteMergeDistributionMode</h4>
<pre class="methodSignature">public&nbsp;<a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a>&nbsp;copyOnWriteMergeDistributionMode()</pre>
</li>
</ul>
<a id="positionDeleteDistributionMode()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>positionDeleteDistributionMode</h4>
<pre class="methodSignature">public&nbsp;<a href="../DistributionMode.html" title="enum in org.apache.iceberg">DistributionMode</a>&nbsp;positionDeleteDistributionMode()</pre>
</li>
</ul>
<a id="useTableDistributionAndOrdering()">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>useTableDistributionAndOrdering</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;useTableDistributionAndOrdering()</pre>
</li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
</div>
</main>
<!-- ========= END OF CLASS DATA ========= -->
<footer role="contentinfo">
<nav role="navigation">
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a id="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</nav>
</footer>
</body>
</html>