blob: e8bcf978bff00f2511affef080bfd4cf9bbbbbbf [file] [log] [blame]
<!DOCTYPE HTML>
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (11.0.12) on Mon Nov 08 18:33:49 PST 2021 -->
<title>ExpireSnapshotsAction</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="dc.created" content="2021-11-08">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<link rel="stylesheet" type="text/css" href="../../../../jquery/jquery-ui.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip/dist/jszip.min.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils.min.js"></script>
<!--[if IE]>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils-ie.min.js"></script>
<![endif]-->
<script type="text/javascript" src="../../../../jquery/jquery-3.5.1.js"></script>
<script type="text/javascript" src="../../../../jquery/jquery-ui.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="ExpireSnapshotsAction";
}
}
catch(err) {
}
//-->
var data = {"i0":42,"i1":42,"i2":42,"i3":42,"i4":42,"i5":42,"i6":42,"i7":42};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"],32:["t6","Deprecated Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
var pathtoroot = "../../../../";
var useModuleDirectories = true;
loadScripts(document, 'script');</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<header role="banner">
<nav role="navigation">
<div class="fixedNav">
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a id="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<ul class="navListSearch">
<li><label for="search">SEARCH:</label>
<input type="text" id="search" value="search" disabled="disabled">
<input type="reset" id="reset" value="reset" disabled="disabled">
</li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
</div>
<div class="navPadding">&nbsp;</div>
<script type="text/javascript"><!--
$('.navPadding').css('padding-top', $('.fixedNav').css("height"));
//-->
</script>
</nav>
</header>
<!-- ======== START OF CLASS DATA ======== -->
<main role="main">
<div class="header">
<div class="subTitle"><span class="packageLabelInType">Package</span>&nbsp;<a href="package-summary.html">org.apache.iceberg.actions</a></div>
<h2 title="Class ExpireSnapshotsAction" class="title">Class ExpireSnapshotsAction</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>org.apache.iceberg.actions.ExpireSnapshotsAction</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<dl>
<dt>All Implemented Interfaces:</dt>
<dd><code><a href="Action.html" title="interface in org.apache.iceberg.actions">Action</a>&lt;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>,&#8203;<a href="ExpireSnapshotsActionResult.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsActionResult</a>&gt;</code></dd>
</dl>
<hr>
<pre>@Deprecated
public class <span class="typeNameLabel">ExpireSnapshotsAction</span>
extends java.lang.Object
implements <a href="Action.html" title="interface in org.apache.iceberg.actions">Action</a>&lt;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>,&#8203;<a href="ExpireSnapshotsActionResult.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsActionResult</a>&gt;</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span>
<div class="deprecationComment">since 0.12.0, will be removed in 0.13.0; use <a href="../spark/actions/BaseExpireSnapshotsSparkAction.html" title="class in org.apache.iceberg.spark.actions"><code>BaseExpireSnapshotsSparkAction</code></a> instead.</div>
</div>
<div class="block">An action which performs the same operation as <a href="../ExpireSnapshots.html" title="interface in org.apache.iceberg"><code>ExpireSnapshots</code></a> but uses Spark
to determine the delta in files between the pre and post-expiration table metadata. All of the same
restrictions of Remove Snapshots also apply to this action.
<p>
This implementation uses the metadata tables for the table being expired to list all Manifest and DataFiles. This
is made into a Dataframe which are anti-joined with the same list read after the expiration. This operation will
require a shuffle so parallelism can be controlled through spark.sql.shuffle.partitions. The expiration is done
locally using a direct call to RemoveSnapshots. The snapshot expiration will be fully committed before any deletes
are issued. Deletes are still performed locally after retrieving the results from the Spark executors.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ========== METHOD SUMMARY =========== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t6" class="tableTab"><span><a href="javascript:show(32);">Deprecated Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colSecond" scope="col">Method</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code><a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#deleteWith(java.util.function.Consumer)">deleteWith</a></span>&#8203;(java.util.function.Consumer&lt;java.lang.String&gt;&nbsp;newDeleteFunc)</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">The Consumer used on files which have been determined to be expired.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code><a href="ExpireSnapshotsActionResult.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsActionResult</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#execute()">execute</a></span>()</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">Executes this action.</div>
</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code><a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#executeDeleteWith(java.util.concurrent.ExecutorService)">executeDeleteWith</a></span>&#8203;(java.util.concurrent.ExecutorService&nbsp;executorService)</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">An executor service used when deleting files.</div>
</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code>org.apache.spark.sql.Dataset&lt;org.apache.spark.sql.Row&gt;</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#expire()">expire</a></span>()</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">Expires snapshots and commits the changes to the table, returning a Dataset of files to delete.</div>
</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code><a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#expireOlderThan(long)">expireOlderThan</a></span>&#8203;(long&nbsp;timestampMillis)</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">Expire all snapshots older than a given timestamp.</div>
</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code><a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#expireSnapshotId(long)">expireSnapshotId</a></span>&#8203;(long&nbsp;expireSnapshotId)</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">A specific snapshot to expire.</div>
</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code><a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#retainLast(int)">retainLast</a></span>&#8203;(int&nbsp;numSnapshots)</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">Retain at least x snapshots when expiring
Identical to <a href="../ExpireSnapshots.html#retainLast(int)"><code>ExpireSnapshots.retainLast(int)</code></a></div>
</td>
</tr>
<tr id="i7" class="rowColor">
<td class="colFirst"><code><a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#streamDeleteResults(boolean)">streamDeleteResults</a></span>&#8203;(boolean&nbsp;stream)</code></th>
<td class="colLast">
<div class="block"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">By default, all files to delete are brought to the driver at once which may be an issue with very long file lists.</div>
</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a id="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
<ul class="blockList">
<li class="blockList"><a id="methods.inherited.from.class.org.apache.iceberg.actions.Action">
<!-- -->
</a>
<h3>Methods inherited from interface&nbsp;org.apache.iceberg.actions.<a href="Action.html" title="interface in org.apache.iceberg.actions">Action</a></h3>
<code><a href="Action.html#option(java.lang.String,java.lang.String)">option</a>, <a href="Action.html#options(java.util.Map)">options</a></code></li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ============ METHOD DETAIL ========== -->
<section role="region">
<ul class="blockList">
<li class="blockList"><a id="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a id="streamDeleteResults(boolean)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>streamDeleteResults</h4>
<pre class="methodSignature">public&nbsp;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>&nbsp;streamDeleteResults&#8203;(boolean&nbsp;stream)</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">By default, all files to delete are brought to the driver at once which may be an issue with very long file lists.
Set this to true to use toLocalIterator if you are running into memory issues when collecting
the list of files to be deleted.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>stream</code> - whether to use toLocalIterator to stream results instead of collect.</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="executeDeleteWith(java.util.concurrent.ExecutorService)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>executeDeleteWith</h4>
<pre class="methodSignature">public&nbsp;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>&nbsp;executeDeleteWith&#8203;(java.util.concurrent.ExecutorService&nbsp;executorService)</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">An executor service used when deleting files. Only used during the local delete phase of this Spark action.
Similar to <a href="../ExpireSnapshots.html#executeDeleteWith(java.util.concurrent.ExecutorService)"><code>ExpireSnapshots.executeDeleteWith(ExecutorService)</code></a></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>executorService</code> - the service to use</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="expireSnapshotId(long)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>expireSnapshotId</h4>
<pre class="methodSignature">public&nbsp;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>&nbsp;expireSnapshotId&#8203;(long&nbsp;expireSnapshotId)</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">A specific snapshot to expire.
Identical to <a href="../ExpireSnapshots.html#expireSnapshotId(long)"><code>ExpireSnapshots.expireSnapshotId(long)</code></a></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>expireSnapshotId</code> - Id of the snapshot to expire</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="expireOlderThan(long)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>expireOlderThan</h4>
<pre class="methodSignature">public&nbsp;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>&nbsp;expireOlderThan&#8203;(long&nbsp;timestampMillis)</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">Expire all snapshots older than a given timestamp.
Identical to <a href="../ExpireSnapshots.html#expireOlderThan(long)"><code>ExpireSnapshots.expireOlderThan(long)</code></a></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>timestampMillis</code> - all snapshots before this time will be expired</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="retainLast(int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>retainLast</h4>
<pre class="methodSignature">public&nbsp;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>&nbsp;retainLast&#8203;(int&nbsp;numSnapshots)</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">Retain at least x snapshots when expiring
Identical to <a href="../ExpireSnapshots.html#retainLast(int)"><code>ExpireSnapshots.retainLast(int)</code></a></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>numSnapshots</code> - number of snapshots to leave</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="deleteWith(java.util.function.Consumer)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>deleteWith</h4>
<pre class="methodSignature">public&nbsp;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>&nbsp;deleteWith&#8203;(java.util.function.Consumer&lt;java.lang.String&gt;&nbsp;newDeleteFunc)</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">The Consumer used on files which have been determined to be expired. By default uses a filesystem delete.
Identical to <a href="../ExpireSnapshots.html#deleteWith(java.util.function.Consumer)"><code>ExpireSnapshots.deleteWith(Consumer)</code></a></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>newDeleteFunc</code> - Consumer which takes a path and deletes it</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="expire()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>expire</h4>
<pre class="methodSignature">public&nbsp;org.apache.spark.sql.Dataset&lt;org.apache.spark.sql.Row&gt;&nbsp;expire()</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block">Expires snapshots and commits the changes to the table, returning a Dataset of files to delete.
<p>
This does not delete data files. To delete data files, run <a href="#execute()"><code>execute()</code></a>.
<p>
This may be called before or after <a href="#execute()"><code>execute()</code></a> is called to return the expired file list.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>a Dataset of files that are no longer referenced by the table</dd>
</dl>
</li>
</ul>
<a id="execute()">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>execute</h4>
<pre class="methodSignature">public&nbsp;<a href="ExpireSnapshotsActionResult.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsActionResult</a>&nbsp;execute()</pre>
<div class="deprecationBlock"><span class="deprecatedLabel">Deprecated.</span></div>
<div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="Action.html#execute()">Action</a></code></span></div>
<div class="block">Executes this action.</div>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="Action.html#execute()">execute</a></code>&nbsp;in interface&nbsp;<code><a href="Action.html" title="interface in org.apache.iceberg.actions">Action</a>&lt;<a href="ExpireSnapshotsAction.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsAction</a>,&#8203;<a href="ExpireSnapshotsActionResult.html" title="class in org.apache.iceberg.actions">ExpireSnapshotsActionResult</a>&gt;</code></dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the result of this action</dd>
</dl>
</li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
</div>
</main>
<!-- ========= END OF CLASS DATA ========= -->
<footer role="contentinfo">
<nav role="navigation">
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a id="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</nav>
</footer>
</body>
</html>