Google Git
Sign in
apache / iceberg / refs/heads/asf-site / . / javadoc / 1.5.0 / org / apache / iceberg / actions / DeleteOrphanFiles.html
blob: 18dd5c55750055ef79fc742f784c114695e83e87 [file] [log] [blame]
<!DOCTYPE HTML>
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc -->
<title>DeleteOrphanFiles</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<link rel="stylesheet" type="text/css" href="../../../../jquery/jquery-ui.min.css" title="Style">
<link rel="stylesheet" type="text/css" href="../../../../jquery-ui.overrides.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip/dist/jszip.min.js"></script>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils.min.js"></script>
<!--[if IE]>
<script type="text/javascript" src="../../../../jquery/jszip-utils/dist/jszip-utils-ie.min.js"></script>
<![endif]-->
<script type="text/javascript" src="../../../../jquery/jquery-3.6.1.min.js"></script>
<script type="text/javascript" src="../../../../jquery/jquery-ui.min.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="DeleteOrphanFiles";
}
}
catch(err) {
}
//-->
var data = {"i0":6,"i1":18,"i2":18,"i3":6,"i4":6,"i5":6,"i6":18};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],4:["t3","Abstract Methods"],16:["t5","Default Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
var pathtoroot = "../../../../";
var useModuleDirectories = true;
loadScripts(document, 'script');</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<header role="banner">
<nav role="navigation">
<div class="fixedNav">
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a id="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<ul class="navListSearch">
<li><label for="search">SEARCH:</label>
<input type="text" id="search" value="search" disabled="disabled">
<input type="reset" id="reset" value="reset" disabled="disabled">
</li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li><a href="#nested.class.summary">Nested</a>&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
</div>
<div class="navPadding">&nbsp;</div>
<script type="text/javascript"><!--
$('.navPadding').css('padding-top', $('.fixedNav').css("height"));
//-->
</script>
</nav>
</header>
<!-- ======== START OF CLASS DATA ======== -->
<main role="main">
<div class="header">
<div class="subTitle"><span class="packageLabelInType">Package</span>&nbsp;<a href="package-summary.html">org.apache.iceberg.actions</a></div>
<h2 title="Interface DeleteOrphanFiles" class="title">Interface DeleteOrphanFiles</h2>
</div>
<div class="contentContainer">
<div class="description">
<ul class="blockList">
<li class="blockList">
<dl>
<dt>All Superinterfaces:</dt>
<dd><code><a href="Action.html" title="interface in org.apache.iceberg.actions">Action</a>&lt;<a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>,&#8203;<a href="DeleteOrphanFiles.Result.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles.Result</a>&gt;</code></dd>
</dl>
<dl>
<dt>All Known Implementing Classes:</dt>
<dd><code><a href="../spark/actions/DeleteOrphanFilesSparkAction.html" title="class in org.apache.iceberg.spark.actions">DeleteOrphanFilesSparkAction</a></code></dd>
</dl>
<hr>
<pre>public interface <span class="typeNameLabel">DeleteOrphanFiles</span>
extends <a href="Action.html" title="interface in org.apache.iceberg.actions">Action</a>&lt;<a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>,&#8203;<a href="DeleteOrphanFiles.Result.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles.Result</a>&gt;</pre>
<div class="block">An action that deletes orphan metadata, data and delete files in a table.
<p>A file is considered orphan if it is not reachable by any valid snapshot. The set of actual
files is built by listing the underlying storage which makes this operation expensive.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ======== NESTED CLASS SUMMARY ======== -->
<section>
<ul class="blockList">
<li class="blockList"><a id="nested.class.summary">
<!-- -->
</a>
<h3>Nested Class Summary</h3>
<table class="memberSummary">
<caption><span>Nested Classes</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colSecond" scope="col">Interface</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="DeleteOrphanFiles.PrefixMismatchMode.html" title="enum in org.apache.iceberg.actions">DeleteOrphanFiles.PrefixMismatchMode</a></span></code></th>
<td class="colLast">
<div class="block">Defines the action behavior when location prefixes (scheme/authority) mismatch.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static interface&nbsp;</code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="DeleteOrphanFiles.Result.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles.Result</a></span></code></th>
<td class="colLast">
<div class="block">The action result that contains a summary of the execution.</div>
</td>
</tr>
</table>
</li>
</ul>
</section>
<!-- ========== METHOD SUMMARY =========== -->
<section>
<ul class="blockList">
<li class="blockList"><a id="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t3" class="tableTab"><span><a href="javascript:show(4);">Abstract Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t5" class="tableTab"><span><a href="javascript:show(16);">Default Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colSecond" scope="col">Method</th>
<th class="colLast" scope="col">Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#deleteWith(java.util.function.Consumer)">deleteWith</a></span>&#8203;(java.util.function.Consumer&lt;java.lang.String&gt;&nbsp;deleteFunc)</code></th>
<td class="colLast">
<div class="block">Passes an alternative delete implementation that will be used for orphan files.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>default <a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#equalAuthorities(java.util.Map)">equalAuthorities</a></span>&#8203;(java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;newEqualAuthorities)</code></th>
<td class="colLast">
<div class="block">Passes authorities that should be considered equal.</div>
</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code>default <a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#equalSchemes(java.util.Map)">equalSchemes</a></span>&#8203;(java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;newEqualSchemes)</code></th>
<td class="colLast">
<div class="block">Passes schemes that should be considered equal.</div>
</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#executeDeleteWith(java.util.concurrent.ExecutorService)">executeDeleteWith</a></span>&#8203;(java.util.concurrent.ExecutorService&nbsp;executorService)</code></th>
<td class="colLast">
<div class="block">Passes an alternative executor service that will be used for removing orphaned files.</div>
</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#location(java.lang.String)">location</a></span>&#8203;(java.lang.String&nbsp;location)</code></th>
<td class="colLast">
<div class="block">Passes a location which should be scanned for orphan files.</div>
</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#olderThan(long)">olderThan</a></span>&#8203;(long&nbsp;olderThanTimestamp)</code></th>
<td class="colLast">
<div class="block">Removes orphan files only if they are older than the given timestamp.</div>
</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code>default <a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a></code></td>
<th class="colSecond" scope="row"><code><span class="memberNameLink"><a href="#prefixMismatchMode(org.apache.iceberg.actions.DeleteOrphanFiles.PrefixMismatchMode)">prefixMismatchMode</a></span>&#8203;(<a href="DeleteOrphanFiles.PrefixMismatchMode.html" title="enum in org.apache.iceberg.actions">DeleteOrphanFiles.PrefixMismatchMode</a>&nbsp;newPrefixMismatchMode)</code></th>
<td class="colLast">
<div class="block">Passes a prefix mismatch mode that determines how this action should handle situations when the
metadata references files that match listed/provided files except for authority/scheme.</div>
</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a id="methods.inherited.from.class.org.apache.iceberg.actions.Action">
<!-- -->
</a>
<h3>Methods inherited from interface&nbsp;org.apache.iceberg.actions.<a href="Action.html" title="interface in org.apache.iceberg.actions">Action</a></h3>
<code><a href="Action.html#execute()">execute</a>, <a href="Action.html#option(java.lang.String,java.lang.String)">option</a>, <a href="Action.html#options(java.util.Map)">options</a></code></li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ============ METHOD DETAIL ========== -->
<section>
<ul class="blockList">
<li class="blockList"><a id="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a id="location(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>location</h4>
<pre class="methodSignature"><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>&nbsp;location&#8203;(java.lang.String&nbsp;location)</pre>
<div class="block">Passes a location which should be scanned for orphan files.
<p>If not set, the root table location will be scanned potentially removing both orphan data
and metadata files.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>location</code> - the location where to look for orphan files</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="olderThan(long)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>olderThan</h4>
<pre class="methodSignature"><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>&nbsp;olderThan&#8203;(long&nbsp;olderThanTimestamp)</pre>
<div class="block">Removes orphan files only if they are older than the given timestamp.
<p>This is a safety measure to avoid removing files that are being added to the table. For
example, there may be a concurrent operation adding new files while this action searches for
orphan files. New files may not be referenced by the metadata yet but they are not orphan.
<p>If not set, defaults to a timestamp 3 days ago.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>olderThanTimestamp</code> - a long timestamp, as returned by <code>System.currentTimeMillis()</code></dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="deleteWith(java.util.function.Consumer)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>deleteWith</h4>
<pre class="methodSignature"><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>&nbsp;deleteWith&#8203;(java.util.function.Consumer&lt;java.lang.String&gt;&nbsp;deleteFunc)</pre>
<div class="block">Passes an alternative delete implementation that will be used for orphan files.
<p>This method allows users to customize the delete function. For example, one may set a custom
delete func and collect all orphan files into a set instead of physically removing them.
<p>If not set, defaults to using the table's <a href="../io/FileIO.html" title="interface in org.apache.iceberg.io"><code>io</code></a>
implementation.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>deleteFunc</code> - a function that will be called to delete files</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="executeDeleteWith(java.util.concurrent.ExecutorService)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>executeDeleteWith</h4>
<pre class="methodSignature"><a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>&nbsp;executeDeleteWith&#8203;(java.util.concurrent.ExecutorService&nbsp;executorService)</pre>
<div class="block">Passes an alternative executor service that will be used for removing orphaned files. This
service will only be used if a custom delete function is provided by <a href="#deleteWith(java.util.function.Consumer)"><code>deleteWith(Consumer)</code></a> or if the FileIO does not <a href="../io/SupportsBulkOperations.html" title="interface in org.apache.iceberg.io"><code>support bulk
deletes</code></a>. Otherwise, parallelism should be controlled by the IO specific <a href="../io/SupportsBulkOperations.html#deleteFiles(java.lang.Iterable)"><code>deleteFiles</code></a> method.
<p>If this method is not called and bulk deletes are not supported, orphaned manifests and data
files will still be deleted in the current thread.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>executorService</code> - the service to use</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="prefixMismatchMode(org.apache.iceberg.actions.DeleteOrphanFiles.PrefixMismatchMode)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>prefixMismatchMode</h4>
<pre class="methodSignature">default&nbsp;<a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>&nbsp;prefixMismatchMode&#8203;(<a href="DeleteOrphanFiles.PrefixMismatchMode.html" title="enum in org.apache.iceberg.actions">DeleteOrphanFiles.PrefixMismatchMode</a>&nbsp;newPrefixMismatchMode)</pre>
<div class="block">Passes a prefix mismatch mode that determines how this action should handle situations when the
metadata references files that match listed/provided files except for authority/scheme.
<p>Possible values are "ERROR", "IGNORE", "DELETE". The default mismatch mode is "ERROR", which
means an exception is thrown whenever there is a mismatch in authority/scheme. It's the
recommended mismatch mode and should be changed only in some rare circumstances. If there is a
mismatch, use <a href="#equalSchemes(java.util.Map)"><code>equalSchemes(Map)</code></a> and <a href="#equalAuthorities(java.util.Map)"><code>equalAuthorities(Map)</code></a> to resolve
conflicts by providing equivalent schemes and authorities. If it is impossible to determine
whether the conflicting authorities/schemes are equal, set the prefix mismatch mode to "IGNORE"
to skip files with mismatches. If you have manually inspected all conflicting
authorities/schemes, provided equivalent schemes/authorities and are absolutely confident the
remaining ones are different, set the prefix mismatch mode to "DELETE" to consider files with
mismatches as orphan. It will be impossible to recover files after deletion, so the "DELETE"
prefix mismatch mode must be used with extreme caution.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>newPrefixMismatchMode</code> - mode for handling prefix mismatches</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="equalSchemes(java.util.Map)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>equalSchemes</h4>
<pre class="methodSignature">default&nbsp;<a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>&nbsp;equalSchemes&#8203;(java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;newEqualSchemes)</pre>
<div class="block">Passes schemes that should be considered equal.
<p>The key may include a comma-separated list of schemes. For instance, Map("s3a,s3,s3n",
"s3").</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>newEqualSchemes</code> - list of equal schemes</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
<a id="equalAuthorities(java.util.Map)">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>equalAuthorities</h4>
<pre class="methodSignature">default&nbsp;<a href="DeleteOrphanFiles.html" title="interface in org.apache.iceberg.actions">DeleteOrphanFiles</a>&nbsp;equalAuthorities&#8203;(java.util.Map&lt;java.lang.String,&#8203;java.lang.String&gt;&nbsp;newEqualAuthorities)</pre>
<div class="block">Passes authorities that should be considered equal.
<p>The key may include a comma-separate list of authorities. For instance, Map("s1name,s2name",
"servicename").</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>newEqualAuthorities</code> - list of equal authorities</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>this for method chaining</dd>
</dl>
</li>
</ul>
</li>
</ul>
</section>
</li>
</ul>
</div>
</div>
</main>
<!-- ========= END OF CLASS DATA ========= -->
<footer role="contentinfo">
<nav role="navigation">
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a id="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a id="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../index-all.html">Index</a></li>
<li><a href="../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../allclasses.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li><a href="#nested.class.summary">Nested</a>&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a id="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</nav>
</footer>
</body>
</html>