blob: 0303b4c271a569552849fa4eb7ed1403d912a645 [file] [log] [blame]
<!DOCTYPE HTML>
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc -->
<title>SparkReadConf</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<link rel="stylesheet" type="text/css" href="../../../../jquery/jquery-ui.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip/dist/jszip.min.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils.min.js"></script>
<!--[if IE]>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils-ie.min.js"></script>
<![endif]-->
<script type="text/javascript" src="../../../../jquery/jquery-3.5.1.js"></script>
<script type="text/javascript" src="../../../../jquery/jquery-ui.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="SparkReadConf";
}
}
catch(err) {
}
//-->
var data = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
var pathtoroot = "../../../../";
var useModuleDirectories = true;
loadScripts(document, 'script');</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<header role="banner">
<nav role="navigation">
<div class="fixedNav">
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a id="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<ul class="navListSearch">
<li><label for="search">SEARCH:</label>
<input type="text" id="search" value="search" disabled="disabled">
<input type="reset" id="reset" value="reset" disabled="disabled">
</li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
</div>
<div class="navPadding">&nbsp;</div>
<script type="text/javascript"><!--
$('.navPadding').css('padding-top', $('.fixedNav').css("height"));
//-->
</script>
</nav>
</header>
<!-- ======== START OF CLASS DATA ======== -->
<main role="main">
<div class="header">
<div class="subTitle"><span class="packageLabelInType">Package</span>&nbsp;<a href="package-summary.html">org.apache.iceberg.spark</a></div>
<h2 title="Class SparkReadConf" class="title">Class SparkReadConf</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>org.apache.iceberg.spark.SparkReadConf</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<hr>
<pre>public class <span class="typeNameLabel">SparkReadConf</span>
extends java.lang.Object</pre>
<div class="block">A class for common Iceberg configs for Spark reads.
<p>
If a config is set at multiple levels, the following order of precedence is used (top to bottom):
<ol>
<li>Read options</li>
<li>Session configuration</li>
<li>Table metadata</li>
</ol>
The most specific value is set in read options and takes precedence over all other configs.
If no read option is provided, this class checks the session configuration for any overrides.
If no applicable value is found in the session configuration, this class uses the table metadata.
<p>
Note this class is NOT meant to be serialized and sent to executors.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="constructor.summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="memberSummary">
<caption><span>Constructors</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Constructor</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tr class="altColor">
<th class="colConstructorName" scope="row"><code><span class="memberNameLink"><a href="#%3Cinit%3E(org.apache.spark.sql.SparkSession,org.apache.iceberg.Table,java.util.Map)">SparkReadConf</a></span>&#8203;(org.apache.spark.sql.SparkSession&nbsp;spark,
<a href="../Table.html" title="interface in org.apache.iceberg">Table</a>&nbsp;table,
java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;readOptions)</code></th>
<td class="colLast">&nbsp;</td>
</tr>
</table>
</li>
</ul>
</section>
<!-- ========== METHOD SUMMARY =========== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colSecond" scope="col">Method</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code>java.lang.Long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#asOfTimestamp()">asOfTimestamp</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#caseSensitive()">caseSensitive</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code>java.lang.Long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#endSnapshotId()">endSnapshotId</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#fileScanTaskSetId()">fileScanTaskSetId</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#handleTimestampWithoutZone()">handleTimestampWithoutZone</a></span>()</code></th>
<td class="colLast">
<div class="block">Enables reading a timestamp without time zone as a timestamp with time zone.</div>
</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#localityEnabled()">localityEnabled</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code>int</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#orcBatchSize()">orcBatchSize</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i7" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#orcVectorizationEnabled()">orcVectorizationEnabled</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i8" class="altColor">
<td class="colFirst"><code>int</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#parquetBatchSize()">parquetBatchSize</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i9" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#parquetVectorizationEnabled()">parquetVectorizationEnabled</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i10" class="altColor">
<td class="colFirst"><code>java.lang.Long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#snapshotId()">snapshotId</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i11" class="rowColor">
<td class="colFirst"><code>int</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#splitLookback()">splitLookback</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i12" class="altColor">
<td class="colFirst"><code>java.lang.Integer</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#splitLookbackOption()">splitLookbackOption</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i13" class="rowColor">
<td class="colFirst"><code>long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#splitOpenFileCost()">splitOpenFileCost</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i14" class="altColor">
<td class="colFirst"><code>java.lang.Long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#splitOpenFileCostOption()">splitOpenFileCostOption</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i15" class="rowColor">
<td class="colFirst"><code>long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#splitSize()">splitSize</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i16" class="altColor">
<td class="colFirst"><code>java.lang.Long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#splitSizeOption()">splitSizeOption</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i17" class="rowColor">
<td class="colFirst"><code>java.lang.Long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#startSnapshotId()">startSnapshotId</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i18" class="altColor">
<td class="colFirst"><code>java.lang.Long</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#streamFromTimestamp()">streamFromTimestamp</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
<tr id="i19" class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#streamingSkipDeleteSnapshots()">streamingSkipDeleteSnapshots</a></span>()</code></th>
<td class="colLast">&nbsp;</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a id="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="constructor.detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a id="&lt;init&gt;(org.apache.spark.sql.SparkSession,org.apache.iceberg.Table,java.util.Map)">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>SparkReadConf</h4>
<pre>public&nbsp;SparkReadConf&#8203;(org.apache.spark.sql.SparkSession&nbsp;spark,
<a href="../Table.html" title="interface in org.apache.iceberg">Table</a>&nbsp;table,
java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;readOptions)</pre>
</li>
</ul>
</li>
</ul>
</section>
<!-- ============ METHOD DETAIL ========== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a id="caseSensitive()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>caseSensitive</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;caseSensitive()</pre>
</li>
</ul>
<a id="localityEnabled()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>localityEnabled</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;localityEnabled()</pre>
</li>
</ul>
<a id="snapshotId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>snapshotId</h4>
<pre class="methodSignature">public&nbsp;java.lang.Long&nbsp;snapshotId()</pre>
</li>
</ul>
<a id="asOfTimestamp()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>asOfTimestamp</h4>
<pre class="methodSignature">public&nbsp;java.lang.Long&nbsp;asOfTimestamp()</pre>
</li>
</ul>
<a id="startSnapshotId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>startSnapshotId</h4>
<pre class="methodSignature">public&nbsp;java.lang.Long&nbsp;startSnapshotId()</pre>
</li>
</ul>
<a id="endSnapshotId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>endSnapshotId</h4>
<pre class="methodSignature">public&nbsp;java.lang.Long&nbsp;endSnapshotId()</pre>
</li>
</ul>
<a id="fileScanTaskSetId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>fileScanTaskSetId</h4>
<pre class="methodSignature">public&nbsp;java.lang.String&nbsp;fileScanTaskSetId()</pre>
</li>
</ul>
<a id="streamingSkipDeleteSnapshots()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>streamingSkipDeleteSnapshots</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;streamingSkipDeleteSnapshots()</pre>
</li>
</ul>
<a id="parquetVectorizationEnabled()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>parquetVectorizationEnabled</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;parquetVectorizationEnabled()</pre>
</li>
</ul>
<a id="parquetBatchSize()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>parquetBatchSize</h4>
<pre class="methodSignature">public&nbsp;int&nbsp;parquetBatchSize()</pre>
</li>
</ul>
<a id="orcVectorizationEnabled()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>orcVectorizationEnabled</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;orcVectorizationEnabled()</pre>
</li>
</ul>
<a id="orcBatchSize()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>orcBatchSize</h4>
<pre class="methodSignature">public&nbsp;int&nbsp;orcBatchSize()</pre>
</li>
</ul>
<a id="splitSizeOption()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>splitSizeOption</h4>
<pre class="methodSignature">public&nbsp;java.lang.Long&nbsp;splitSizeOption()</pre>
</li>
</ul>
<a id="splitSize()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>splitSize</h4>
<pre class="methodSignature">public&nbsp;long&nbsp;splitSize()</pre>
</li>
</ul>
<a id="splitLookbackOption()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>splitLookbackOption</h4>
<pre class="methodSignature">public&nbsp;java.lang.Integer&nbsp;splitLookbackOption()</pre>
</li>
</ul>
<a id="splitLookback()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>splitLookback</h4>
<pre class="methodSignature">public&nbsp;int&nbsp;splitLookback()</pre>
</li>
</ul>
<a id="splitOpenFileCostOption()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>splitOpenFileCostOption</h4>
<pre class="methodSignature">public&nbsp;java.lang.Long&nbsp;splitOpenFileCostOption()</pre>
</li>
</ul>
<a id="splitOpenFileCost()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>splitOpenFileCost</h4>
<pre class="methodSignature">public&nbsp;long&nbsp;splitOpenFileCost()</pre>
</li>
</ul>
<a id="handleTimestampWithoutZone()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>handleTimestampWithoutZone</h4>
<pre class="methodSignature">public&nbsp;boolean&nbsp;handleTimestampWithoutZone()</pre>
<div class="block">Enables reading a timestamp without time zone as a timestamp with time zone.
<p>
Generally, this is not safe as a timestamp without time zone is supposed to represent the wall-clock time,
i.e. no matter the reader/writer timezone 3PM should always be read as 3PM,
but a timestamp with time zone represents instant semantics, i.e. the timestamp
is adjusted so that the corresponding time in the reader timezone is displayed.
<p>
When set to false (default), an exception must be thrown while reading a timestamp without time zone.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>boolean indicating if reading timestamps without timezone is allowed</dd>
</dl>
</li>
</ul>
<a id="streamFromTimestamp()">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>streamFromTimestamp</h4>
<pre class="methodSignature">public&nbsp;java.lang.Long&nbsp;streamFromTimestamp()</pre>
</li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
</div>
</main>
<!-- ========= END OF CLASS DATA ========= -->
<footer role="contentinfo">
<nav role="navigation">
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a id="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</nav>
</footer>
</body>
</html>