Google Git
Sign in
apache / spark-website / bf09c083b54ac9a00c74c79b8b38346b2fdd0a59 / . / site / docs / 3.3.3 / api / java / org / apache / spark / ShuffleStatus.html
blob: 44b7067a7f1a4c36512ddca38ef1192ca9f43ba3 [file] [log] [blame]
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (1.8.0_382) on Fri Aug 04 18:26:25 UTC 2023 -->
<title>ShuffleStatus (Spark 3.3.3 JavaDoc)</title>
<meta name="date" content="2023-08-04">
<link rel="stylesheet" type="text/css" href="../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="ShuffleStatus (Spark 3.3.3 JavaDoc)";
}
}
catch(err) {
}
//-->
var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../index-all.html">Index</a></li>
<li><a href="../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../org/apache/spark/ShuffleDependency.html" title="class in org.apache.spark"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../org/apache/spark/SimpleFutureAction.html" title="class in org.apache.spark"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../index.html?org/apache/spark/ShuffleStatus.html" target="_top">Frames</a></li>
<li><a href="ShuffleStatus.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">org.apache.spark</div>
<h2 title="Class ShuffleStatus" class="title">Class ShuffleStatus</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>Object</li>
<li>
<ul class="inheritance">
<li>org.apache.spark.ShuffleStatus</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<dl>
<dt>All Implemented Interfaces:</dt>
<dd>org.apache.spark.internal.Logging</dd>
</dl>
<hr>
<br>
<pre>public class <span class="typeNameLabel">ShuffleStatus</span>
extends Object
implements org.apache.spark.internal.Logging</pre>
<div class="block">Helper class used by the <code>MapOutputTrackerMaster</code> to perform bookkeeping for a single
ShuffleMapStage.
<p>
This class maintains a mapping from map index to <code>MapStatus</code>. It also maintains a cache of
serialized map statuses in order to speed up tasks' requests for map output statuses.
<p>
All public methods of this class are thread-safe.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ======== NESTED CLASS SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="nested.class.summary">
<!-- -->
</a>
<h3>Nested Class Summary</h3>
<ul class="blockList">
<li class="blockList"><a name="nested.classes.inherited.from.class.org.apache.spark.internal.Logging">
<!-- -->
</a>
<h3>Nested classes/interfaces inherited from interface&nbsp;org.apache.spark.internal.Logging</h3>
<code>org.apache.spark.internal.Logging.SparkShellLoggingFilter</code></li>
</ul>
</li>
</ul>
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation">
<caption><span>Constructors</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colOne" scope="col">Constructor and Description</th>
</tr>
<tr class="altColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#ShuffleStatus-int-int-">ShuffleStatus</a></span>(int&nbsp;numPartitions,
int&nbsp;numReducers)</code>&nbsp;</td>
</tr>
</table>
</li>
</ul>
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#addMapOutput-int-org.apache.spark.scheduler.MapStatus-">addMapOutput</a></span>(int&nbsp;mapIndex,
<a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>&nbsp;status)</code>
<div class="block">Register a map output.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#addMergeResult-int-org.apache.spark.scheduler.MergeStatus-">addMergeResult</a></span>(int&nbsp;reduceId,
org.apache.spark.scheduler.MergeStatus&nbsp;status)</code>
<div class="block">Register a merge result.</div>
</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code>scala.collection.Seq&lt;Object&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#findMissingPartitions--">findMissingPartitions</a></span>()</code>
<div class="block">Returns the sequence of partition ids that are missing (i.e.</div>
</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code>scala.collection.Seq&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#getShufflePushMergerLocations--">getShufflePushMergerLocations</a></span>()</code>&nbsp;</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#hasCachedSerializedBroadcast--">hasCachedSerializedBroadcast</a></span>()</code>&nbsp;</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#invalidateSerializedMapOutputStatusCache--">invalidateSerializedMapOutputStatusCache</a></span>()</code>
<div class="block">Clears the cached serialized map output statuses.</div>
</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#invalidateSerializedMergeOutputStatusCache--">invalidateSerializedMergeOutputStatusCache</a></span>()</code>
<div class="block">Clears the cached serialized merge result statuses.</div>
</td>
</tr>
<tr id="i7" class="rowColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#mapStatuses--">mapStatuses</a></span>()</code>
<div class="block">MapStatus for each partition.</div>
</td>
</tr>
<tr id="i8" class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#mapStatusesDeleted--">mapStatusesDeleted</a></span>()</code>
<div class="block">Keep the previous deleted MapStatus for recovery.</div>
</td>
</tr>
<tr id="i9" class="rowColor">
<td class="colFirst"><code>org.apache.spark.scheduler.MergeStatus[]</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#mergeStatuses--">mergeStatuses</a></span>()</code>
<div class="block">MergeStatus for each shuffle partition when push-based shuffle is enabled.</div>
</td>
</tr>
<tr id="i10" class="altColor">
<td class="colFirst"><code>int</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#numAvailableMapOutputs--">numAvailableMapOutputs</a></span>()</code>
<div class="block">Number of partitions that have shuffle map outputs.</div>
</td>
</tr>
<tr id="i11" class="rowColor">
<td class="colFirst"><code>int</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#numAvailableMergeResults--">numAvailableMergeResults</a></span>()</code>
<div class="block">Number of shuffle partitions that have already been merge finalized when push-based
is enabled.</div>
</td>
</tr>
<tr id="i12" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#registerShuffleMergerLocations-scala.collection.Seq-">registerShuffleMergerLocations</a></span>(scala.collection.Seq&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;&nbsp;shuffleMergers)</code>&nbsp;</td>
</tr>
<tr id="i13" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#removeMapOutput-int-org.apache.spark.storage.BlockManagerId-">removeMapOutput</a></span>(int&nbsp;mapIndex,
<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</code>
<div class="block">Remove the map output which was served by the specified block manager.</div>
</td>
</tr>
<tr id="i14" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#removeMergeResult-int-org.apache.spark.storage.BlockManagerId-">removeMergeResult</a></span>(int&nbsp;reduceId,
<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</code>
<div class="block">Remove the merge result which was served by the specified block manager.</div>
</td>
</tr>
<tr id="i15" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#removeMergeResultsByFilter-scala.Function1-">removeMergeResultsByFilter</a></span>(scala.Function1&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,Object&gt;&nbsp;f)</code>
<div class="block">Removes all shuffle merge result which satisfies the filter.</div>
</td>
</tr>
<tr id="i16" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#removeOutputsByFilter-scala.Function1-">removeOutputsByFilter</a></span>(scala.Function1&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,Object&gt;&nbsp;f)</code>
<div class="block">Removes all shuffle outputs which satisfies the filter.</div>
</td>
</tr>
<tr id="i17" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#removeOutputsOnExecutor-java.lang.String-">removeOutputsOnExecutor</a></span>(String&nbsp;execId)</code>
<div class="block">Removes all map outputs associated with the specified executor.</div>
</td>
</tr>
<tr id="i18" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#removeOutputsOnHost-java.lang.String-">removeOutputsOnHost</a></span>(String&nbsp;host)</code>
<div class="block">Removes all shuffle outputs associated with this host.</div>
</td>
</tr>
<tr id="i19" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#removeShuffleMergerLocations--">removeShuffleMergerLocations</a></span>()</code>&nbsp;</td>
</tr>
<tr id="i20" class="altColor">
<td class="colFirst"><code>scala.Tuple2&lt;byte[],byte[]&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#serializedMapAndMergeStatus-org.apache.spark.broadcast.BroadcastManager-boolean-int-org.apache.spark.SparkConf-">serializedMapAndMergeStatus</a></span>(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</code>
<div class="block">Serializes the mapStatuses and mergeStatuses array into an efficient compressed format.</div>
</td>
</tr>
<tr id="i21" class="rowColor">
<td class="colFirst"><code>byte[]</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#serializedMapStatus-org.apache.spark.broadcast.BroadcastManager-boolean-int-org.apache.spark.SparkConf-">serializedMapStatus</a></span>(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</code>
<div class="block">Serializes the mapStatuses array into an efficient compressed format.</div>
</td>
</tr>
<tr id="i22" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#updateMapOutput-long-org.apache.spark.storage.BlockManagerId-">updateMapOutput</a></span>(long&nbsp;mapId,
<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</code>
<div class="block">Update the map output location (e.g.</div>
</td>
</tr>
<tr id="i23" class="rowColor">
<td class="colFirst"><code>&lt;T&gt;&nbsp;T</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#withMapStatuses-scala.Function1-">withMapStatuses</a></span>(scala.Function1&lt;<a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[],T&gt;&nbsp;f)</code>
<div class="block">Helper function which provides thread-safe access to the mapStatuses array.</div>
</td>
</tr>
<tr id="i24" class="altColor">
<td class="colFirst"><code>&lt;T&gt;&nbsp;T</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../org/apache/spark/ShuffleStatus.html#withMergeStatuses-scala.Function1-">withMergeStatuses</a></span>(scala.Function1&lt;org.apache.spark.scheduler.MergeStatus[],T&gt;&nbsp;f)</code>&nbsp;</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;Object</h3>
<code>equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.org.apache.spark.internal.Logging">
<!-- -->
</a>
<h3>Methods inherited from interface&nbsp;org.apache.spark.internal.Logging</h3>
<code>$init$, initializeForcefully, initializeLogIfNecessary, initializeLogIfNecessary, initializeLogIfNecessary$default$2, initLock, isTraceEnabled, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning, org$apache$spark$internal$Logging$$log__$eq, org$apache$spark$internal$Logging$$log_, uninitialize</code></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a name="ShuffleStatus-int-int-">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>ShuffleStatus</h4>
<pre>public&nbsp;ShuffleStatus(int&nbsp;numPartitions,
int&nbsp;numReducers)</pre>
</li>
</ul>
</li>
</ul>
<!-- ============ METHOD DETAIL ========== -->
<ul class="blockList">
<li class="blockList"><a name="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a name="mapStatuses--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>mapStatuses</h4>
<pre>public&nbsp;<a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]&nbsp;mapStatuses()</pre>
<div class="block">MapStatus for each partition. The index of the array is the map partition id.
Each value in the array is the MapStatus for a partition, or null if the partition
is not available. Even though in theory a task may run multiple times (due to speculation,
stage retries, etc.), in practice the likelihood of a map output being available at multiple
locations is so small that we choose to ignore that case and store only a single location
for each output.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="mapStatusesDeleted--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>mapStatusesDeleted</h4>
<pre>public&nbsp;<a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]&nbsp;mapStatusesDeleted()</pre>
<div class="block">Keep the previous deleted MapStatus for recovery.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="mergeStatuses--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>mergeStatuses</h4>
<pre>public&nbsp;org.apache.spark.scheduler.MergeStatus[]&nbsp;mergeStatuses()</pre>
<div class="block">MergeStatus for each shuffle partition when push-based shuffle is enabled. The index of the
array is the shuffle partition id (reduce id). Each value in the array is the MergeStatus for
a shuffle partition, or null if not available. When push-based shuffle is enabled, this array
provides a reducer oriented view of the shuffle status specifically for the results of
merging shuffle partition blocks into per-partition merged shuffle files.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="addMapOutput-int-org.apache.spark.scheduler.MapStatus-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addMapOutput</h4>
<pre>public&nbsp;void&nbsp;addMapOutput(int&nbsp;mapIndex,
<a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>&nbsp;status)</pre>
<div class="block">Register a map output. If there is already a registered location for the map output then it
will be replaced by the new location.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>mapIndex</code> - (undocumented)</dd>
<dd><code>status</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="updateMapOutput-long-org.apache.spark.storage.BlockManagerId-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>updateMapOutput</h4>
<pre>public&nbsp;void&nbsp;updateMapOutput(long&nbsp;mapId,
<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</pre>
<div class="block">Update the map output location (e.g. during migration).</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>mapId</code> - (undocumented)</dd>
<dd><code>bmAddress</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="removeMapOutput-int-org.apache.spark.storage.BlockManagerId-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>removeMapOutput</h4>
<pre>public&nbsp;void&nbsp;removeMapOutput(int&nbsp;mapIndex,
<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</pre>
<div class="block">Remove the map output which was served by the specified block manager.
This is a no-op if there is no registered map output or if the registered output is from a
different block manager.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>mapIndex</code> - (undocumented)</dd>
<dd><code>bmAddress</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="addMergeResult-int-org.apache.spark.scheduler.MergeStatus-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addMergeResult</h4>
<pre>public&nbsp;void&nbsp;addMergeResult(int&nbsp;reduceId,
org.apache.spark.scheduler.MergeStatus&nbsp;status)</pre>
<div class="block">Register a merge result.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>reduceId</code> - (undocumented)</dd>
<dd><code>status</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="registerShuffleMergerLocations-scala.collection.Seq-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>registerShuffleMergerLocations</h4>
<pre>public&nbsp;void&nbsp;registerShuffleMergerLocations(scala.collection.Seq&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;&nbsp;shuffleMergers)</pre>
</li>
</ul>
<a name="removeShuffleMergerLocations--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>removeShuffleMergerLocations</h4>
<pre>public&nbsp;void&nbsp;removeShuffleMergerLocations()</pre>
</li>
</ul>
<a name="removeMergeResult-int-org.apache.spark.storage.BlockManagerId-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>removeMergeResult</h4>
<pre>public&nbsp;void&nbsp;removeMergeResult(int&nbsp;reduceId,
<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</pre>
<div class="block">Remove the merge result which was served by the specified block manager.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>reduceId</code> - (undocumented)</dd>
<dd><code>bmAddress</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="removeOutputsOnHost-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>removeOutputsOnHost</h4>
<pre>public&nbsp;void&nbsp;removeOutputsOnHost(String&nbsp;host)</pre>
<div class="block">Removes all shuffle outputs associated with this host. Note that this will also remove
outputs which are served by an external shuffle server (if one exists).</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>host</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="removeOutputsOnExecutor-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>removeOutputsOnExecutor</h4>
<pre>public&nbsp;void&nbsp;removeOutputsOnExecutor(String&nbsp;execId)</pre>
<div class="block">Removes all map outputs associated with the specified executor. Note that this will also
remove outputs which are served by an external shuffle server (if one exists), as they are
still registered with that execId.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>execId</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="removeOutputsByFilter-scala.Function1-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>removeOutputsByFilter</h4>
<pre>public&nbsp;void&nbsp;removeOutputsByFilter(scala.Function1&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,Object&gt;&nbsp;f)</pre>
<div class="block">Removes all shuffle outputs which satisfies the filter. Note that this will also
remove outputs which are served by an external shuffle server (if one exists).</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>f</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="removeMergeResultsByFilter-scala.Function1-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>removeMergeResultsByFilter</h4>
<pre>public&nbsp;void&nbsp;removeMergeResultsByFilter(scala.Function1&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,Object&gt;&nbsp;f)</pre>
<div class="block">Removes all shuffle merge result which satisfies the filter.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>f</code> - (undocumented)</dd>
</dl>
</li>
</ul>
<a name="numAvailableMapOutputs--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>numAvailableMapOutputs</h4>
<pre>public&nbsp;int&nbsp;numAvailableMapOutputs()</pre>
<div class="block">Number of partitions that have shuffle map outputs.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="numAvailableMergeResults--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>numAvailableMergeResults</h4>
<pre>public&nbsp;int&nbsp;numAvailableMergeResults()</pre>
<div class="block">Number of shuffle partitions that have already been merge finalized when push-based
is enabled.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="findMissingPartitions--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>findMissingPartitions</h4>
<pre>public&nbsp;scala.collection.Seq&lt;Object&gt;&nbsp;findMissingPartitions()</pre>
<div class="block">Returns the sequence of partition ids that are missing (i.e. needs to be computed).</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="serializedMapStatus-org.apache.spark.broadcast.BroadcastManager-boolean-int-org.apache.spark.SparkConf-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>serializedMapStatus</h4>
<pre>public&nbsp;byte[]&nbsp;serializedMapStatus(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</pre>
<div class="block">Serializes the mapStatuses array into an efficient compressed format. See the comments on
<code>MapOutputTracker.serializeOutputStatuses()</code> for more details on the serialization format.
<p>
This method is designed to be called multiple times and implements caching in order to speed
up subsequent requests. If the cache is empty and multiple threads concurrently attempt to
serialize the map statuses then serialization will only be performed in a single thread and
all other threads will block until the cache is populated.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>broadcastManager</code> - (undocumented)</dd>
<dd><code>isLocal</code> - (undocumented)</dd>
<dd><code>minBroadcastSize</code> - (undocumented)</dd>
<dd><code>conf</code> - (undocumented)</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="serializedMapAndMergeStatus-org.apache.spark.broadcast.BroadcastManager-boolean-int-org.apache.spark.SparkConf-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>serializedMapAndMergeStatus</h4>
<pre>public&nbsp;scala.Tuple2&lt;byte[],byte[]&gt;&nbsp;serializedMapAndMergeStatus(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</pre>
<div class="block">Serializes the mapStatuses and mergeStatuses array into an efficient compressed format.
See the comments on <code>MapOutputTracker.serializeOutputStatuses()</code> for more details
on the serialization format.
<p>
This method is designed to be called multiple times and implements caching in order to speed
up subsequent requests. If the cache is empty and multiple threads concurrently attempt to
serialize the statuses array then serialization will only be performed in a single thread and
all other threads will block until the cache is populated.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>broadcastManager</code> - (undocumented)</dd>
<dd><code>isLocal</code> - (undocumented)</dd>
<dd><code>minBroadcastSize</code> - (undocumented)</dd>
<dd><code>conf</code> - (undocumented)</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="hasCachedSerializedBroadcast--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>hasCachedSerializedBroadcast</h4>
<pre>public&nbsp;boolean&nbsp;hasCachedSerializedBroadcast()</pre>
</li>
</ul>
<a name="withMapStatuses-scala.Function1-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withMapStatuses</h4>
<pre>public&nbsp;&lt;T&gt;&nbsp;T&nbsp;withMapStatuses(scala.Function1&lt;<a href="../../../org/apache/spark/scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[],T&gt;&nbsp;f)</pre>
<div class="block">Helper function which provides thread-safe access to the mapStatuses array.
The function should NOT mutate the array.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>f</code> - (undocumented)</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>(undocumented)</dd>
</dl>
</li>
</ul>
<a name="withMergeStatuses-scala.Function1-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>withMergeStatuses</h4>
<pre>public&nbsp;&lt;T&gt;&nbsp;T&nbsp;withMergeStatuses(scala.Function1&lt;org.apache.spark.scheduler.MergeStatus[],T&gt;&nbsp;f)</pre>
</li>
</ul>
<a name="getShufflePushMergerLocations--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getShufflePushMergerLocations</h4>
<pre>public&nbsp;scala.collection.Seq&lt;<a href="../../../org/apache/spark/storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;&nbsp;getShufflePushMergerLocations()</pre>
</li>
</ul>
<a name="invalidateSerializedMapOutputStatusCache--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>invalidateSerializedMapOutputStatusCache</h4>
<pre>public&nbsp;void&nbsp;invalidateSerializedMapOutputStatusCache()</pre>
<div class="block">Clears the cached serialized map output statuses.</div>
</li>
</ul>
<a name="invalidateSerializedMergeOutputStatusCache--">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>invalidateSerializedMergeOutputStatusCache</h4>
<pre>public&nbsp;void&nbsp;invalidateSerializedMergeOutputStatusCache()</pre>
<div class="block">Clears the cached serialized merge result statuses.</div>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../index-all.html">Index</a></li>
<li><a href="../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../org/apache/spark/ShuffleDependency.html" title="class in org.apache.spark"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../org/apache/spark/SimpleFutureAction.html" title="class in org.apache.spark"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../index.html?org/apache/spark/ShuffleStatus.html" target="_top">Frames</a></li>
<li><a href="ShuffleStatus.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
<script defer="defer" type="text/javascript" src="../../../lib/jquery.js"></script><script defer="defer" type="text/javascript" src="../../../lib/api-javadocs.js"></script></body>
</html>