Google Git
Sign in
apache / spark-website / 74943c8625877cef547882f29b430d612b6f6a45 / . / site / docs / 4.0.0-preview1 / api / java / org / apache / spark / ShuffleStatus.html
blob: 15621c43154d44aa41337e4c165265f747a451ee [file] [log] [blame]
<!DOCTYPE HTML>
<html lang="en">
<head>
<!-- Generated by javadoc (17) on Tue May 28 17:21:12 UTC 2024 -->
<title>ShuffleStatus (Spark 4.0.0-preview1 JavaDoc)</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="dc.created" content="2024-05-28">
<meta name="description" content="declaration: package: org.apache.spark, class: ShuffleStatus">
<meta name="generator" content="javadoc/ClassWriterImpl">
<link rel="stylesheet" type="text/css" href="../../../stylesheet.css" title="Style">
<link rel="stylesheet" type="text/css" href="../../../script-dir/jquery-ui.min.css" title="Style">
<link rel="stylesheet" type="text/css" href="../../../jquery-ui.overrides.css" title="Style">
<script type="text/javascript" src="../../../script.js"></script>
<script type="text/javascript" src="../../../script-dir/jquery-3.6.1.min.js"></script>
<script type="text/javascript" src="../../../script-dir/jquery-ui.min.js"></script>
</head>
<body class="class-declaration-page">
<script type="text/javascript">var evenRowColor = "even-row-color";
var oddRowColor = "odd-row-color";
var tableTab = "table-tab";
var activeTableTab = "active-table-tab";
var pathtoroot = "../../../";
loadScripts(document, 'script');</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<div class="flex-box">
<header role="banner" class="flex-header">
<nav role="navigation">
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="top-nav" id="navbar-top">
<div class="skip-nav"><a href="#skip-navbar-top" title="Skip navigation links">Skip navigation links</a></div>
<ul id="navbar-top-firstrow" class="nav-list" title="Navigation">
<li><a href="../../../index.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="nav-bar-cell1-rev">Class</li>
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../index-all.html">Index</a></li>
<li><a href="../../../help-doc.html#class">Help</a></li>
</ul>
</div>
<div class="sub-nav">
<div>
<ul class="sub-nav-list">
<li>Summary:&nbsp;</li>
<li><a href="#nested-class-summary">Nested</a>&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor-summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method-summary">Method</a></li>
</ul>
<ul class="sub-nav-list">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor-detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method-detail">Method</a></li>
</ul>
</div>
<div class="nav-list-search"><label for="search-input">SEARCH:</label>
<input type="text" id="search-input" value="search" disabled="disabled">
<input type="reset" id="reset-button" value="reset" disabled="disabled">
</div>
</div>
<!-- ========= END OF TOP NAVBAR ========= -->
<span class="skip-nav" id="skip-navbar-top"></span></nav>
</header>
<div class="flex-content">
<main role="main">
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="sub-title"><span class="package-label-in-type">Package</span>&nbsp;<a href="package-summary.html">org.apache.spark</a></div>
<h1 title="Class ShuffleStatus" class="title">Class ShuffleStatus</h1>
</div>
<div class="inheritance" title="Inheritance Tree"><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>
<div class="inheritance">org.apache.spark.ShuffleStatus</div>
</div>
<section class="class-description" id="class-description">
<dl class="notes">
<dt>All Implemented Interfaces:</dt>
<dd><code>org.apache.spark.internal.Logging</code></dd>
</dl>
<hr>
<div class="type-signature"><span class="modifiers">public class </span><span class="element-name type-name-label">ShuffleStatus</span>
<span class="extends-implements">extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>
implements org.apache.spark.internal.Logging</span></div>
<div class="block">Helper class used by the <code>MapOutputTrackerMaster</code> to perform bookkeeping for a single
ShuffleMapStage.
<p>
This class maintains a mapping from map index to <code>MapStatus</code>. It also maintains a cache of
serialized map statuses in order to speed up tasks' requests for map output statuses.
<p>
All public methods of this class are thread-safe.</div>
</section>
<section class="summary">
<ul class="summary-list">
<!-- ======== NESTED CLASS SUMMARY ======== -->
<li>
<section class="nested-class-summary" id="nested-class-summary">
<h2>Nested Class Summary</h2>
<div class="inherited-list">
<h2 id="nested-classes-inherited-from-class-org.apache.spark.internal.Logging">Nested classes/interfaces inherited from interface&nbsp;org.apache.spark.internal.Logging</h2>
<code>org.apache.spark.internal.Logging.LogStringContext, org.apache.spark.internal.Logging.SparkShellLoggingFilter</code></div>
</section>
</li>
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<li>
<section class="constructor-summary" id="constructor-summary">
<h2>Constructor Summary</h2>
<div class="caption"><span>Constructors</span></div>
<div class="summary-table two-column-summary">
<div class="table-header col-first">Constructor</div>
<div class="table-header col-last">Description</div>
<div class="col-constructor-name even-row-color"><code><a href="#%3Cinit%3E(int,int)" class="member-name-link">ShuffleStatus</a><wbr>(int&nbsp;numPartitions,
int&nbsp;numReducers)</code></div>
<div class="col-last even-row-color">&nbsp;</div>
</div>
</section>
</li>
<!-- ========== METHOD SUMMARY =========== -->
<li>
<section class="method-summary" id="method-summary">
<h2>Method Summary</h2>
<div id="method-summary-table">
<div class="table-tabs" role="tablist" aria-orientation="horizontal"><button id="method-summary-table-tab0" role="tab" aria-selected="true" aria-controls="method-summary-table.tabpanel" tabindex="0" onkeydown="switchTab(event)" onclick="show('method-summary-table', 'method-summary-table', 3)" class="active-table-tab">All Methods</button><button id="method-summary-table-tab2" role="tab" aria-selected="false" aria-controls="method-summary-table.tabpanel" tabindex="-1" onkeydown="switchTab(event)" onclick="show('method-summary-table', 'method-summary-table-tab2', 3)" class="table-tab">Instance Methods</button><button id="method-summary-table-tab4" role="tab" aria-selected="false" aria-controls="method-summary-table.tabpanel" tabindex="-1" onkeydown="switchTab(event)" onclick="show('method-summary-table', 'method-summary-table-tab4', 3)" class="table-tab">Concrete Methods</button></div>
<div id="method-summary-table.tabpanel" role="tabpanel">
<div class="summary-table three-column-summary" aria-labelledby="method-summary-table-tab0">
<div class="table-header col-first">Modifier and Type</div>
<div class="table-header col-second">Method</div>
<div class="table-header col-last">Description</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#addMapOutput(int,org.apache.spark.scheduler.MapStatus)" class="member-name-link">addMapOutput</a><wbr>(int&nbsp;mapIndex,
<a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>&nbsp;status)</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Register a map output.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#addMergeResult(int,org.apache.spark.scheduler.MergeStatus)" class="member-name-link">addMergeResult</a><wbr>(int&nbsp;reduceId,
org.apache.spark.scheduler.MergeStatus&nbsp;status)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Register a merge result.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>scala.collection.immutable.Seq&lt;<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>&gt;</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#findMissingPartitions()" class="member-name-link">findMissingPartitions</a>()</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Returns the sequence of partition ids that are missing (i.e.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>scala.Option&lt;<a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>&gt;</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#getMapStatus(long)" class="member-name-link">getMapStatus</a><wbr>(long&nbsp;mapId)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Get the map output that corresponding to a given mapId.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>scala.collection.immutable.Seq&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#getShufflePushMergerLocations()" class="member-name-link">getShufflePushMergerLocations</a>()</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">&nbsp;</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>boolean</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#hasCachedSerializedBroadcast()" class="member-name-link">hasCachedSerializedBroadcast</a>()</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">&nbsp;</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#invalidateSerializedMapOutputStatusCache()" class="member-name-link">invalidateSerializedMapOutputStatusCache</a>()</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Clears the cached serialized map output statuses.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#invalidateSerializedMergeOutputStatusCache()" class="member-name-link">invalidateSerializedMergeOutputStatusCache</a>()</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Clears the cached serialized merge result statuses.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#mapStatuses()" class="member-name-link">mapStatuses</a>()</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">MapStatus for each partition.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#mapStatusesDeleted()" class="member-name-link">mapStatusesDeleted</a>()</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Keep the previous deleted MapStatus for recovery.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>org.apache.spark.scheduler.MergeStatus[]</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#mergeStatuses()" class="member-name-link">mergeStatuses</a>()</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">MergeStatus for each shuffle partition when push-based shuffle is enabled.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>int</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#numAvailableMapOutputs()" class="member-name-link">numAvailableMapOutputs</a>()</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Number of partitions that have shuffle map outputs.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>int</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#numAvailableMergeResults()" class="member-name-link">numAvailableMergeResults</a>()</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Number of shuffle partitions that have already been merge finalized when push-based
is enabled.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#registerShuffleMergerLocations(scala.collection.immutable.Seq)" class="member-name-link">registerShuffleMergerLocations</a><wbr>(scala.collection.immutable.Seq&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;&nbsp;shuffleMergers)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">&nbsp;</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#removeMapOutput(int,org.apache.spark.storage.BlockManagerId)" class="member-name-link">removeMapOutput</a><wbr>(int&nbsp;mapIndex,
<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Remove the map output which was served by the specified block manager.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#removeMergeResult(int,org.apache.spark.storage.BlockManagerId)" class="member-name-link">removeMergeResult</a><wbr>(int&nbsp;reduceId,
<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Remove the merge result which was served by the specified block manager.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#removeMergeResultsByFilter(scala.Function1)" class="member-name-link">removeMergeResultsByFilter</a><wbr>(scala.Function1&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,<wbr><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>&gt;&nbsp;f)</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Removes all shuffle merge result which satisfies the filter.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#removeOutputsByFilter(scala.Function1)" class="member-name-link">removeOutputsByFilter</a><wbr>(scala.Function1&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,<wbr><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>&gt;&nbsp;f)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Removes all shuffle outputs which satisfies the filter.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#removeOutputsOnExecutor(java.lang.String)" class="member-name-link">removeOutputsOnExecutor</a><wbr>(<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a>&nbsp;execId)</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Removes all map outputs associated with the specified executor.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#removeOutputsOnHost(java.lang.String)" class="member-name-link">removeOutputsOnHost</a><wbr>(<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a>&nbsp;host)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Removes all shuffle outputs associated with this host.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#removeShuffleMergerLocations()" class="member-name-link">removeShuffleMergerLocations</a>()</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">&nbsp;</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>scala.Tuple2&lt;byte[],<wbr>byte[]&gt;</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#serializedMapAndMergeStatus(org.apache.spark.broadcast.BroadcastManager,boolean,int,org.apache.spark.SparkConf)" class="member-name-link">serializedMapAndMergeStatus</a><wbr>(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Serializes the mapStatuses and mergeStatuses array into an efficient compressed format.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>byte[]</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#serializedMapStatus(org.apache.spark.broadcast.BroadcastManager,boolean,int,org.apache.spark.SparkConf)" class="member-name-link">serializedMapStatus</a><wbr>(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Serializes the mapStatuses array into an efficient compressed format.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#updateMapOutput(long,org.apache.spark.storage.BlockManagerId)" class="member-name-link">updateMapOutput</a><wbr>(long&nbsp;mapId,
<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Update the map output location (e.g.</div>
</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>&lt;T&gt;&nbsp;T</code></div>
<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#withMapStatuses(scala.Function1)" class="member-name-link">withMapStatuses</a><wbr>(scala.Function1&lt;<a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[],<wbr>T&gt;&nbsp;f)</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Helper function which provides thread-safe access to the mapStatuses array.</div>
</div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>&lt;T&gt;&nbsp;T</code></div>
<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#withMergeStatuses(scala.Function1)" class="member-name-link">withMergeStatuses</a><wbr>(scala.Function1&lt;org.apache.spark.scheduler.MergeStatus[],<wbr>T&gt;&nbsp;f)</code></div>
<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">&nbsp;</div>
</div>
</div>
</div>
<div class="inherited-list">
<h3 id="methods-inherited-from-class-Object">Methods inherited from class&nbsp;java.lang.<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a></h3>
<code><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#equals(java.lang.Object)" title="class or interface in java.lang" class="external-link">equals</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#getClass()" title="class or interface in java.lang" class="external-link">getClass</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#hashCode()" title="class or interface in java.lang" class="external-link">hashCode</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#notify()" title="class or interface in java.lang" class="external-link">notify</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#notifyAll()" title="class or interface in java.lang" class="external-link">notifyAll</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#toString()" title="class or interface in java.lang" class="external-link">toString</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#wait()" title="class or interface in java.lang" class="external-link">wait</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#wait(long)" title="class or interface in java.lang" class="external-link">wait</a>, <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html#wait(long,int)" title="class or interface in java.lang" class="external-link">wait</a></code></div>
<div class="inherited-list">
<h3 id="methods-inherited-from-class-org.apache.spark.internal.Logging">Methods inherited from interface&nbsp;org.apache.spark.internal.Logging</h3>
<code>initializeForcefully, initializeLogIfNecessary, initializeLogIfNecessary, initializeLogIfNecessary$default$2, isTraceEnabled, log, logDebug, logDebug, logDebug, logDebug, logError, logError, logError, logError, logInfo, logInfo, logInfo, logInfo, logName, LogStringContext, logTrace, logTrace, logTrace, logTrace, logWarning, logWarning, logWarning, logWarning, org$apache$spark$internal$Logging$$log_, org$apache$spark$internal$Logging$$log__$eq, withLogContext</code></div>
</section>
</li>
</ul>
</section>
<section class="details">
<ul class="details-list">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<li>
<section class="constructor-details" id="constructor-detail">
<h2>Constructor Details</h2>
<ul class="member-list">
<li>
<section class="detail" id="&lt;init&gt;(int,int)">
<h3>ShuffleStatus</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="element-name">ShuffleStatus</span><wbr><span class="parameters">(int&nbsp;numPartitions,
int&nbsp;numReducers)</span></div>
</section>
</li>
</ul>
</section>
</li>
<!-- ============ METHOD DETAIL ========== -->
<li>
<section class="method-details" id="method-detail">
<h2>Method Details</h2>
<ul class="member-list">
<li>
<section class="detail" id="mapStatuses()">
<h3>mapStatuses</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type"><a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]</span>&nbsp;<span class="element-name">mapStatuses</span>()</div>
<div class="block">MapStatus for each partition. The index of the array is the map partition id.
Each value in the array is the MapStatus for a partition, or null if the partition
is not available. Even though in theory a task may run multiple times (due to speculation,
stage retries, etc.), in practice the likelihood of a map output being available at multiple
locations is so small that we choose to ignore that case and store only a single location
for each output.</div>
<dl class="notes">
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="mapStatusesDeleted()">
<h3>mapStatusesDeleted</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type"><a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[]</span>&nbsp;<span class="element-name">mapStatusesDeleted</span>()</div>
<div class="block">Keep the previous deleted MapStatus for recovery.</div>
<dl class="notes">
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="mergeStatuses()">
<h3>mergeStatuses</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">org.apache.spark.scheduler.MergeStatus[]</span>&nbsp;<span class="element-name">mergeStatuses</span>()</div>
<div class="block">MergeStatus for each shuffle partition when push-based shuffle is enabled. The index of the
array is the shuffle partition id (reduce id). Each value in the array is the MergeStatus for
a shuffle partition, or null if not available. When push-based shuffle is enabled, this array
provides a reducer oriented view of the shuffle status specifically for the results of
merging shuffle partition blocks into per-partition merged shuffle files.</div>
<dl class="notes">
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="addMapOutput(int,org.apache.spark.scheduler.MapStatus)">
<h3>addMapOutput</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">addMapOutput</span><wbr><span class="parameters">(int&nbsp;mapIndex,
<a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>&nbsp;status)</span></div>
<div class="block">Register a map output. If there is already a registered location for the map output then it
will be replaced by the new location.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>mapIndex</code> - (undocumented)</dd>
<dd><code>status</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="getMapStatus(long)">
<h3>getMapStatus</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">scala.Option&lt;<a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>&gt;</span>&nbsp;<span class="element-name">getMapStatus</span><wbr><span class="parameters">(long&nbsp;mapId)</span></div>
<div class="block">Get the map output that corresponding to a given mapId.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>mapId</code> - (undocumented)</dd>
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="updateMapOutput(long,org.apache.spark.storage.BlockManagerId)">
<h3>updateMapOutput</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">updateMapOutput</span><wbr><span class="parameters">(long&nbsp;mapId,
<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</span></div>
<div class="block">Update the map output location (e.g. during migration).</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>mapId</code> - (undocumented)</dd>
<dd><code>bmAddress</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="removeMapOutput(int,org.apache.spark.storage.BlockManagerId)">
<h3>removeMapOutput</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">removeMapOutput</span><wbr><span class="parameters">(int&nbsp;mapIndex,
<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</span></div>
<div class="block">Remove the map output which was served by the specified block manager.
This is a no-op if there is no registered map output or if the registered output is from a
different block manager.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>mapIndex</code> - (undocumented)</dd>
<dd><code>bmAddress</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="addMergeResult(int,org.apache.spark.scheduler.MergeStatus)">
<h3>addMergeResult</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">addMergeResult</span><wbr><span class="parameters">(int&nbsp;reduceId,
org.apache.spark.scheduler.MergeStatus&nbsp;status)</span></div>
<div class="block">Register a merge result.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>reduceId</code> - (undocumented)</dd>
<dd><code>status</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="registerShuffleMergerLocations(scala.collection.immutable.Seq)">
<h3>registerShuffleMergerLocations</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">registerShuffleMergerLocations</span><wbr><span class="parameters">(scala.collection.immutable.Seq&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;&nbsp;shuffleMergers)</span></div>
</section>
</li>
<li>
<section class="detail" id="removeShuffleMergerLocations()">
<h3>removeShuffleMergerLocations</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">removeShuffleMergerLocations</span>()</div>
</section>
</li>
<li>
<section class="detail" id="removeMergeResult(int,org.apache.spark.storage.BlockManagerId)">
<h3>removeMergeResult</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">removeMergeResult</span><wbr><span class="parameters">(int&nbsp;reduceId,
<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&nbsp;bmAddress)</span></div>
<div class="block">Remove the merge result which was served by the specified block manager.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>reduceId</code> - (undocumented)</dd>
<dd><code>bmAddress</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="removeOutputsOnHost(java.lang.String)">
<h3>removeOutputsOnHost</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">removeOutputsOnHost</span><wbr><span class="parameters">(<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a>&nbsp;host)</span></div>
<div class="block">Removes all shuffle outputs associated with this host. Note that this will also remove
outputs which are served by an external shuffle server (if one exists).</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>host</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="removeOutputsOnExecutor(java.lang.String)">
<h3>removeOutputsOnExecutor</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">removeOutputsOnExecutor</span><wbr><span class="parameters">(<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a>&nbsp;execId)</span></div>
<div class="block">Removes all map outputs associated with the specified executor. Note that this will also
remove outputs which are served by an external shuffle server (if one exists), as they are
still registered with that execId.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>execId</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="removeOutputsByFilter(scala.Function1)">
<h3>removeOutputsByFilter</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">removeOutputsByFilter</span><wbr><span class="parameters">(scala.Function1&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,<wbr><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>&gt;&nbsp;f)</span></div>
<div class="block">Removes all shuffle outputs which satisfies the filter. Note that this will also
remove outputs which are served by an external shuffle server (if one exists).</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>f</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="removeMergeResultsByFilter(scala.Function1)">
<h3>removeMergeResultsByFilter</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">removeMergeResultsByFilter</span><wbr><span class="parameters">(scala.Function1&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>,<wbr><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>&gt;&nbsp;f)</span></div>
<div class="block">Removes all shuffle merge result which satisfies the filter.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>f</code> - (undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="numAvailableMapOutputs()">
<h3>numAvailableMapOutputs</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">int</span>&nbsp;<span class="element-name">numAvailableMapOutputs</span>()</div>
<div class="block">Number of partitions that have shuffle map outputs.</div>
<dl class="notes">
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="numAvailableMergeResults()">
<h3>numAvailableMergeResults</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">int</span>&nbsp;<span class="element-name">numAvailableMergeResults</span>()</div>
<div class="block">Number of shuffle partitions that have already been merge finalized when push-based
is enabled.</div>
<dl class="notes">
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="findMissingPartitions()">
<h3>findMissingPartitions</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">scala.collection.immutable.Seq&lt;<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a>&gt;</span>&nbsp;<span class="element-name">findMissingPartitions</span>()</div>
<div class="block">Returns the sequence of partition ids that are missing (i.e. needs to be computed).</div>
<dl class="notes">
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="serializedMapStatus(org.apache.spark.broadcast.BroadcastManager,boolean,int,org.apache.spark.SparkConf)">
<h3>serializedMapStatus</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">byte[]</span>&nbsp;<span class="element-name">serializedMapStatus</span><wbr><span class="parameters">(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</span></div>
<div class="block">Serializes the mapStatuses array into an efficient compressed format. See the comments on
<code>MapOutputTracker.serializeOutputStatuses()</code> for more details on the serialization format.
<p>
This method is designed to be called multiple times and implements caching in order to speed
up subsequent requests. If the cache is empty and multiple threads concurrently attempt to
serialize the map statuses then serialization will only be performed in a single thread and
all other threads will block until the cache is populated.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>broadcastManager</code> - (undocumented)</dd>
<dd><code>isLocal</code> - (undocumented)</dd>
<dd><code>minBroadcastSize</code> - (undocumented)</dd>
<dd><code>conf</code> - (undocumented)</dd>
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="serializedMapAndMergeStatus(org.apache.spark.broadcast.BroadcastManager,boolean,int,org.apache.spark.SparkConf)">
<h3>serializedMapAndMergeStatus</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">scala.Tuple2&lt;byte[],<wbr>byte[]&gt;</span>&nbsp;<span class="element-name">serializedMapAndMergeStatus</span><wbr><span class="parameters">(org.apache.spark.broadcast.BroadcastManager&nbsp;broadcastManager,
boolean&nbsp;isLocal,
int&nbsp;minBroadcastSize,
<a href="SparkConf.html" title="class in org.apache.spark">SparkConf</a>&nbsp;conf)</span></div>
<div class="block">Serializes the mapStatuses and mergeStatuses array into an efficient compressed format.
See the comments on <code>MapOutputTracker.serializeOutputStatuses()</code> for more details
on the serialization format.
<p>
This method is designed to be called multiple times and implements caching in order to speed
up subsequent requests. If the cache is empty and multiple threads concurrently attempt to
serialize the statuses array then serialization will only be performed in a single thread and
all other threads will block until the cache is populated.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>broadcastManager</code> - (undocumented)</dd>
<dd><code>isLocal</code> - (undocumented)</dd>
<dd><code>minBroadcastSize</code> - (undocumented)</dd>
<dd><code>conf</code> - (undocumented)</dd>
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="hasCachedSerializedBroadcast()">
<h3>hasCachedSerializedBroadcast</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">boolean</span>&nbsp;<span class="element-name">hasCachedSerializedBroadcast</span>()</div>
</section>
</li>
<li>
<section class="detail" id="withMapStatuses(scala.Function1)">
<h3>withMapStatuses</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="type-parameters">&lt;T&gt;</span>&nbsp;<span class="return-type">T</span>&nbsp;<span class="element-name">withMapStatuses</span><wbr><span class="parameters">(scala.Function1&lt;<a href="scheduler/MapStatus.html" title="interface in org.apache.spark.scheduler">MapStatus</a>[],<wbr>T&gt;&nbsp;f)</span></div>
<div class="block">Helper function which provides thread-safe access to the mapStatuses array.
The function should NOT mutate the array.</div>
<dl class="notes">
<dt>Parameters:</dt>
<dd><code>f</code> - (undocumented)</dd>
<dt>Returns:</dt>
<dd>(undocumented)</dd>
</dl>
</section>
</li>
<li>
<section class="detail" id="withMergeStatuses(scala.Function1)">
<h3>withMergeStatuses</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="type-parameters">&lt;T&gt;</span>&nbsp;<span class="return-type">T</span>&nbsp;<span class="element-name">withMergeStatuses</span><wbr><span class="parameters">(scala.Function1&lt;org.apache.spark.scheduler.MergeStatus[],<wbr>T&gt;&nbsp;f)</span></div>
</section>
</li>
<li>
<section class="detail" id="getShufflePushMergerLocations()">
<h3>getShufflePushMergerLocations</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">scala.collection.immutable.Seq&lt;<a href="storage/BlockManagerId.html" title="class in org.apache.spark.storage">BlockManagerId</a>&gt;</span>&nbsp;<span class="element-name">getShufflePushMergerLocations</span>()</div>
</section>
</li>
<li>
<section class="detail" id="invalidateSerializedMapOutputStatusCache()">
<h3>invalidateSerializedMapOutputStatusCache</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">invalidateSerializedMapOutputStatusCache</span>()</div>
<div class="block">Clears the cached serialized map output statuses.</div>
</section>
</li>
<li>
<section class="detail" id="invalidateSerializedMergeOutputStatusCache()">
<h3>invalidateSerializedMergeOutputStatusCache</h3>
<div class="member-signature"><span class="modifiers">public</span>&nbsp;<span class="return-type">void</span>&nbsp;<span class="element-name">invalidateSerializedMergeOutputStatusCache</span>()</div>
<div class="block">Clears the cached serialized merge result statuses.</div>
</section>
</li>
</ul>
</section>
</li>
</ul>
</section>
<!-- ========= END OF CLASS DATA ========= -->
</main>
</div>
</div>
<script defer="defer" type="text/javascript" src="../../../lib/jquery.js"></script><script defer="defer" type="text/javascript" src="../../../lib/api-javadocs.js"></script></body>
</html>