| <!DOCTYPE html ><html><head><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" name="viewport"/><title>Spark 4.0.0-preview1 ScalaDoc - org.apache.spark</title><meta content="Spark 4.0.0 - preview1 ScalaDoc - org.apache.spark" name="description"/><meta content="Spark 4.0.0 preview1 ScalaDoc org.apache.spark" name="keywords"/><meta http-equiv="content-type" content="text/html; charset=UTF-8"/><link href="../../../lib/index.css" media="screen" type="text/css" rel="stylesheet"/><link href="../../../lib/template.css" media="screen" type="text/css" rel="stylesheet"/><link href="../../../lib/print.css" media="print" type="text/css" rel="stylesheet"/><link href="../../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css"/><script type="text/javascript" src="../../../lib/jquery.min.js"></script><script type="text/javascript" src="../../../lib/index.js"></script><script type="text/javascript" src="../../../index.js"></script><script type="text/javascript" src="../../../lib/scheduler.js"></script><script type="text/javascript" src="../../../lib/template.js"></script><script type="text/javascript">/* this variable can be used by the JS to determine the path to the root document */ |
| var toRoot = '../../../';</script></head><body><div id="search"><span id="doc-title">Spark 4.0.0-preview1 ScalaDoc<span id="doc-version"></span></span> <span class="close-results"><span class="left"><</span> Back</span><div id="textfilter"><span class="input"><input autocapitalize="none" placeholder="Search" id="index-input" type="text" accesskey="/"/><i class="clear material-icons"></i><i id="search-icon" class="material-icons"></i></span></div></div><div id="search-results"><div id="search-progress"><div id="progress-fill"></div></div><div id="results-content"><div id="entity-results"></div><div id="member-results"></div></div></div><div id="content-scroll-container" style="-webkit-overflow-scrolling: touch;"><div id="content-container" style="-webkit-overflow-scrolling: touch;"><div id="subpackage-spacer"><div id="packages"><h1>Packages</h1><ul><li class="indented0 " name="_root_.root" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="_root_" class="anchorToMember"></a><a id="root:_root_" class="anchorToMember"></a> <span class="permalink"><a href="../../../index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="../../../index.html" title=""><span class="name">root</span></a></span><div class="fullcomment"><dl class="attributes block"><dt>Definition Classes</dt><dd><a href="../../../index.html" name="_root_" id="_root_" class="extype">root</a></dd></dl></div></li><li class="indented1 " name="_root_.org" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="org" class="anchorToMember"></a><a id="org:org" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="../../index.html" title=""><span class="name">org</span></a></span><div class="fullcomment"><dl class="attributes block"><dt>Definition Classes</dt><dd><a href="../../../index.html" name="_root_" id="_root_" class="extype">root</a></dd></dl></div></li><li class="indented2 " name="org.apache" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="apache" class="anchorToMember"></a><a id="apache:apache" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="../index.html" title=""><span class="name">apache</span></a></span><div class="fullcomment"><dl class="attributes block"><dt>Definition Classes</dt><dd><a href="../../index.html" name="org" id="org" class="extype">org</a></dd></dl></div></li><li class="indented3 " name="org.apache.parquet" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="parquet" class="anchorToMember"></a><a id="parquet:parquet" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/parquet/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="../parquet/index.html" title=""><span class="name">parquet</span></a></span><div class="fullcomment"><dl class="attributes block"><dt>Definition Classes</dt><dd><a href="../index.html" name="org.apache" id="org.apache" class="extype">apache</a></dd></dl></div></li><li class="indented3 current" name="org.apache.spark" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="spark" class="anchorToMember"></a><a id="spark:spark" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><span class="name">spark</span></span><p class="shortcomment cmt">Core Spark functionality.</p><div class="fullcomment"><div class="comment cmt"><p>Core Spark functionality. <a href="SparkContext.html" name="org.apache.spark.SparkContext" id="org.apache.spark.SparkContext" class="extype">org.apache.spark.SparkContext</a> serves as the main entry point to |
| Spark, while <a href="rdd/RDD.html" name="org.apache.spark.rdd.RDD" id="org.apache.spark.rdd.RDD" class="extype">org.apache.spark.rdd.RDD</a> is the data type representing a distributed collection, |
| and provides most parallel operations.</p><p>In addition, <a href="rdd/PairRDDFunctions.html" name="org.apache.spark.rdd.PairRDDFunctions" id="org.apache.spark.rdd.PairRDDFunctions" class="extype">org.apache.spark.rdd.PairRDDFunctions</a> contains operations available only on RDDs |
| of key-value pairs, such as <code>groupByKey</code> and <code>join</code>; <a href="rdd/DoubleRDDFunctions.html" name="org.apache.spark.rdd.DoubleRDDFunctions" id="org.apache.spark.rdd.DoubleRDDFunctions" class="extype">org.apache.spark.rdd.DoubleRDDFunctions</a> |
| contains operations available only on RDDs of Doubles; and |
| <a href="rdd/SequenceFileRDDFunctions.html" name="org.apache.spark.rdd.SequenceFileRDDFunctions" id="org.apache.spark.rdd.SequenceFileRDDFunctions" class="extype">org.apache.spark.rdd.SequenceFileRDDFunctions</a> contains operations available on RDDs that can |
| be saved as SequenceFiles. These operations are automatically available on any RDD of the right |
| type (e.g. RDD[(Int, Int)] through implicit conversions.</p><p>Java programmers should reference the <a href="api/java/index.html" name="org.apache.spark.api.java" id="org.apache.spark.api.java" class="extype">org.apache.spark.api.java</a> package |
| for Spark programming APIs in Java.</p><p>Classes and methods marked with <span class="experimental badge" style="float: none;"> |
| Experimental</span> are user-facing features which have not been officially adopted by the |
| Spark project. These are subject to change or removal in minor releases.</p><p>Classes and methods marked with <span class="developer badge" style="float: none;"> |
| Developer API</span> are intended for advanced users want to extend Spark through lower |
| level interfaces. These are subject to changes or removal in minor releases. |
| </p></div><dl class="attributes block"><dt>Definition Classes</dt><dd><a href="../index.html" name="org.apache" id="org.apache" class="extype">apache</a></dd></dl></div></li><li class="indented4 " name="org.apache.spark.api" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="api" class="anchorToMember"></a><a id="api:api" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/api/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="api/index.html" title=""><span class="name">api</span></a></span></li><li class="indented4 " name="org.apache.spark.broadcast" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="broadcast" class="anchorToMember"></a><a id="broadcast:broadcast" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/broadcast/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="broadcast/index.html" title="Spark's broadcast variables, used to broadcast immutable datasets to all nodes."><span class="name">broadcast</span></a></span><p class="shortcomment cmt">Spark's broadcast variables, used to broadcast immutable datasets to all nodes.</p></li><li class="indented4 " name="org.apache.spark.graphx" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="graphx" class="anchorToMember"></a><a id="graphx:graphx" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/graphx/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="graphx/index.html" title="ALPHA COMPONENT GraphX is a graph processing framework built on top of Spark."><span class="name">graphx</span></a></span><p class="shortcomment cmt"><span class="badge" style="float: right;">ALPHA COMPONENT</span> |
| GraphX is a graph processing framework built on top of Spark.</p></li><li class="indented4 " name="org.apache.spark.input" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="input" class="anchorToMember"></a><a id="input:input" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/input/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="input/index.html" title=""><span class="name">input</span></a></span></li><li class="indented4 " name="org.apache.spark.io" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="io" class="anchorToMember"></a><a id="io:io" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/io/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="io/index.html" title="IO codecs used for compression."><span class="name">io</span></a></span><p class="shortcomment cmt">IO codecs used for compression.</p><div class="fullcomment"><div class="comment cmt"><p>IO codecs used for compression. See <a href="io/CompressionCodec.html" name="org.apache.spark.io.CompressionCodec" id="org.apache.spark.io.CompressionCodec" class="extype">org.apache.spark.io.CompressionCodec</a>. |
| </p></div></div></li><li class="indented4 " name="org.apache.spark.launcher" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="launcher" class="anchorToMember"></a><a id="launcher:launcher" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/launcher/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="launcher/index.html" title=""><span class="name">launcher</span></a></span></li><li class="indented4 " name="org.apache.spark.mapred" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="mapred" class="anchorToMember"></a><a id="mapred:mapred" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/mapred/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="mapred/index.html" title=""><span class="name">mapred</span></a></span></li><li class="indented4 " name="org.apache.spark.metrics" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="metrics" class="anchorToMember"></a><a id="metrics:metrics" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/metrics/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="metrics/index.html" title=""><span class="name">metrics</span></a></span></li><li class="indented4 " name="org.apache.spark.ml" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="ml" class="anchorToMember"></a><a id="ml:ml" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ml/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="ml/index.html" title="DataFrame-based machine learning APIs to let users quickly assemble and configure practical machine learning pipelines."><span class="name">ml</span></a></span><p class="shortcomment cmt">DataFrame-based machine learning APIs to let users quickly assemble and configure practical |
| machine learning pipelines.</p></li><li class="indented4 " name="org.apache.spark.mllib" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="mllib" class="anchorToMember"></a><a id="mllib:mllib" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/mllib/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="mllib/index.html" title="RDD-based machine learning APIs (in maintenance mode)."><span class="name">mllib</span></a></span><p class="shortcomment cmt">RDD-based machine learning APIs (in maintenance mode).</p><div class="fullcomment"><div class="comment cmt"><p>RDD-based machine learning APIs (in maintenance mode).</p><p>The <code>spark.mllib</code> package is in maintenance mode as of the Spark 2.0.0 release to encourage |
| migration to the DataFrame-based APIs under the <a href="ml/index.html" name="org.apache.spark.ml" id="org.apache.spark.ml" class="extype">org.apache.spark.ml</a> package. |
| While in maintenance mode,</p><ul><li>no new features in the RDD-based <code>spark.mllib</code> package will be accepted, unless they block |
| implementing new features in the DataFrame-based <code>spark.ml</code> package;</li><li>bug fixes in the RDD-based APIs will still be accepted.</li></ul><p>The developers will continue adding more features to the DataFrame-based APIs in the 2.x series |
| to reach feature parity with the RDD-based APIs. |
| And once we reach feature parity, this package will be deprecated. |
| </p></div><dl class="attributes block"><dt>See also</dt><dd><span class="cmt"><p><a href="https://issues.apache.org/jira/browse/SPARK-4591">SPARK-4591</a> to track |
| the progress of feature parity</p></span></dd></dl></div></li><li class="indented4 " name="org.apache.spark.partial" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="partial" class="anchorToMember"></a><a id="partial:partial" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/partial/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="partial/index.html" title="Support for approximate results."><span class="name">partial</span></a></span><p class="shortcomment cmt">Support for approximate results.</p><div class="fullcomment"><div class="comment cmt"><p>Support for approximate results. This provides convenient api and also implementation for |
| approximate calculation. |
| </p></div><dl class="attributes block"><dt>See also</dt><dd><span class="cmt"><p><a href="rdd/RDD.html#countApprox(timeout:Long,confidence:Double):org.apache.spark.partial.PartialResult[org.apache.spark.partial.BoundedDouble]" name="org.apache.spark.rdd.RDD#countApprox" id="org.apache.spark.rdd.RDD#countApprox" class="extmbr">org.apache.spark.rdd.RDD.countApprox</a></p></span></dd></dl></div></li><li class="indented4 " name="org.apache.spark.paths" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="paths" class="anchorToMember"></a><a id="paths:paths" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/paths/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="paths/index.html" title=""><span class="name">paths</span></a></span></li><li class="indented4 " name="org.apache.spark.rdd" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="rdd" class="anchorToMember"></a><a id="rdd:rdd" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/rdd/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="rdd/index.html" title="Provides several RDD implementations."><span class="name">rdd</span></a></span><p class="shortcomment cmt">Provides several RDD implementations.</p><div class="fullcomment"><div class="comment cmt"><p>Provides several RDD implementations. See <a href="rdd/RDD.html" name="org.apache.spark.rdd.RDD" id="org.apache.spark.rdd.RDD" class="extype">org.apache.spark.rdd.RDD</a>. |
| </p></div></div></li><li class="indented4 " name="org.apache.spark.resource" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="resource" class="anchorToMember"></a><a id="resource:resource" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/resource/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="resource/index.html" title=""><span class="name">resource</span></a></span></li><li class="indented4 " name="org.apache.spark.scheduler" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="scheduler" class="anchorToMember"></a><a id="scheduler:scheduler" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/scheduler/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="scheduler/index.html" title="Spark's scheduling components."><span class="name">scheduler</span></a></span><p class="shortcomment cmt">Spark's scheduling components.</p><div class="fullcomment"><div class="comment cmt"><p>Spark's scheduling components. This includes the <code>org.apache.spark.scheduler.DAGScheduler</code> and |
| lower level <code>org.apache.spark.scheduler.TaskScheduler</code>. |
| </p></div></div></li><li class="indented4 " name="org.apache.spark.security" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="security" class="anchorToMember"></a><a id="security:security" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/security/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="security/index.html" title=""><span class="name">security</span></a></span></li><li class="indented4 " name="org.apache.spark.serializer" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="serializer" class="anchorToMember"></a><a id="serializer:serializer" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/serializer/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="serializer/index.html" title="Pluggable serializers for RDD and shuffle data."><span class="name">serializer</span></a></span><p class="shortcomment cmt">Pluggable serializers for RDD and shuffle data.</p><div class="fullcomment"><div class="comment cmt"><p>Pluggable serializers for RDD and shuffle data. |
| </p></div><dl class="attributes block"><dt>See also</dt><dd><span class="cmt"><p><a href="serializer/Serializer.html" name="org.apache.spark.serializer.Serializer" id="org.apache.spark.serializer.Serializer" class="extype">org.apache.spark.serializer.Serializer</a></p></span></dd></dl></div></li><li class="indented4 " name="org.apache.spark.shuffle" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="shuffle" class="anchorToMember"></a><a id="shuffle:shuffle" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/shuffle/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="shuffle/index.html" title=""><span class="name">shuffle</span></a></span></li><li class="indented4 " name="org.apache.spark.sql" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="sql" class="anchorToMember"></a><a id="sql:sql" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/sql/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="sql/index.html" title="Allows the execution of relational queries, including those expressed in SQL using Spark."><span class="name">sql</span></a></span><p class="shortcomment cmt">Allows the execution of relational queries, including those expressed in SQL using Spark.</p></li><li class="indented4 " name="org.apache.spark.status" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="status" class="anchorToMember"></a><a id="status:status" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/status/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="status/index.html" title=""><span class="name">status</span></a></span></li><li class="indented4 " name="org.apache.spark.storage" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="storage" class="anchorToMember"></a><a id="storage:storage" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/storage/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="storage/index.html" title=""><span class="name">storage</span></a></span></li><li class="indented4 " name="org.apache.spark.streaming" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="streaming" class="anchorToMember"></a><a id="streaming:streaming" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/streaming/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="streaming/index.html" title="Spark Streaming functionality."><span class="name">streaming</span></a></span><p class="shortcomment cmt">Spark Streaming functionality.</p><div class="fullcomment"><div class="comment cmt"><p>Spark Streaming functionality. <a href="streaming/StreamingContext.html" name="org.apache.spark.streaming.StreamingContext" id="org.apache.spark.streaming.StreamingContext" class="extype">org.apache.spark.streaming.StreamingContext</a> serves as the main |
| entry point to Spark Streaming, while <a href="streaming/dstream/DStream.html" name="org.apache.spark.streaming.dstream.DStream" id="org.apache.spark.streaming.dstream.DStream" class="extype">org.apache.spark.streaming.dstream.DStream</a> is the data |
| type representing a continuous sequence of RDDs, representing a continuous stream of data.</p><p>In addition, <a href="streaming/dstream/PairDStreamFunctions.html" name="org.apache.spark.streaming.dstream.PairDStreamFunctions" id="org.apache.spark.streaming.dstream.PairDStreamFunctions" class="extype">org.apache.spark.streaming.dstream.PairDStreamFunctions</a> contains operations |
| available only on DStreams |
| of key-value pairs, such as <code>groupByKey</code> and <code>reduceByKey</code>. These operations are automatically |
| available on any DStream of the right type (e.g. DStream[(Int, Int)] through implicit |
| conversions.</p><p>For the Java API of Spark Streaming, take a look at the |
| <a href="streaming/api/java/JavaStreamingContext.html" name="org.apache.spark.streaming.api.java.JavaStreamingContext" id="org.apache.spark.streaming.api.java.JavaStreamingContext" class="extype">org.apache.spark.streaming.api.java.JavaStreamingContext</a> which serves as the entry point, and |
| the <a href="streaming/api/java/JavaDStream.html" name="org.apache.spark.streaming.api.java.JavaDStream" id="org.apache.spark.streaming.api.java.JavaDStream" class="extype">org.apache.spark.streaming.api.java.JavaDStream</a> and the |
| <a href="streaming/api/java/JavaPairDStream.html" name="org.apache.spark.streaming.api.java.JavaPairDStream" id="org.apache.spark.streaming.api.java.JavaPairDStream" class="extype">org.apache.spark.streaming.api.java.JavaPairDStream</a> which have the DStream functionality. |
| </p></div></div></li><li class="indented4 " name="org.apache.spark.types" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="types" class="anchorToMember"></a><a id="types:types" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/types/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="types/index.html" title=""><span class="name">types</span></a></span></li><li class="indented4 " name="org.apache.spark.ui" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="ui" class="anchorToMember"></a><a id="ui:ui" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ui/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="ui/index.html" title=""><span class="name">ui</span></a></span></li><li class="indented4 " name="org.apache.spark.unsafe" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="unsafe" class="anchorToMember"></a><a id="unsafe:unsafe" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/unsafe/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="unsafe/index.html" title=""><span class="name">unsafe</span></a></span></li><li class="indented4 " name="org.apache.spark.util" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="util" class="anchorToMember"></a><a id="util:util" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/util/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="util/index.html" title="Spark utilities."><span class="name">util</span></a></span><p class="shortcomment cmt">Spark utilities.</p></li><li class="current-entities indented3"><span class="separator"></span> <a href="Aggregator.html" title=":: DeveloperApi :: A set of functions used to aggregate data." class="class"></a><a href="Aggregator.html" title=":: DeveloperApi :: A set of functions used to aggregate data.">Aggregator</a></li><li class="current-entities indented3"><a href="BarrierTaskContext$.html" title="" class="object"></a> <a href="BarrierTaskContext.html" title=":: Experimental :: A TaskContext with extra contextual info and tooling for tasks in a barrier stage." class="class"></a><a href="BarrierTaskContext.html" title=":: Experimental :: A TaskContext with extra contextual info and tooling for tasks in a barrier stage.">BarrierTaskContext</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="BarrierTaskInfo.html" title=":: Experimental :: Carries all task infos of a barrier task." class="class"></a><a href="BarrierTaskInfo.html" title=":: Experimental :: Carries all task infos of a barrier task.">BarrierTaskInfo</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="ComplexFutureAction.html" title="A FutureAction for actions that could trigger multiple Spark jobs." class="class"></a><a href="ComplexFutureAction.html" title="A FutureAction for actions that could trigger multiple Spark jobs.">ComplexFutureAction</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="ContextAwareIterator.html" title=":: DeveloperApi :: A TaskContext aware iterator." class="class"></a><a href="ContextAwareIterator.html" title=":: DeveloperApi :: A TaskContext aware iterator.">ContextAwareIterator</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="Dependency.html" title=":: DeveloperApi :: Base class for dependencies." class="class"></a><a href="Dependency.html" title=":: DeveloperApi :: Base class for dependencies.">Dependency</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="ErrorClassesJsonReader.html" title="A reader to load error information from one or more JSON files." class="class"></a><a href="ErrorClassesJsonReader.html" title="A reader to load error information from one or more JSON files.">ErrorClassesJsonReader</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="ExceptionFailure.html" title=":: DeveloperApi :: Task failed due to a runtime exception." class="class"></a><a href="ExceptionFailure.html" title=":: DeveloperApi :: Task failed due to a runtime exception.">ExceptionFailure</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="ExecutorLostFailure.html" title=":: DeveloperApi :: The task failed because the executor that it was running on was lost." class="class"></a><a href="ExecutorLostFailure.html" title=":: DeveloperApi :: The task failed because the executor that it was running on was lost.">ExecutorLostFailure</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="FetchFailed.html" title=":: DeveloperApi :: Task failed to fetch shuffle data from a remote node." class="class"></a><a href="FetchFailed.html" title=":: DeveloperApi :: Task failed to fetch shuffle data from a remote node.">FetchFailed</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="FutureAction.html" title="A future for the result of an action to support cancellation." class="trait"></a><a href="FutureAction.html" title="A future for the result of an action to support cancellation.">FutureAction</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="HashPartitioner.html" title="A org.apache.spark.Partitioner that implements hash-based partitioning using Java's Object.hashCode." class="class"></a><a href="HashPartitioner.html" title="A org.apache.spark.Partitioner that implements hash-based partitioning using Java's Object.hashCode.">HashPartitioner</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="InterruptibleIterator.html" title=":: DeveloperApi :: An iterator that wraps around an existing iterator to provide task killing functionality." class="class"></a><a href="InterruptibleIterator.html" title=":: DeveloperApi :: An iterator that wraps around an existing iterator to provide task killing functionality.">InterruptibleIterator</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="JobExecutionStatus.html" title="" class="class"></a><a href="JobExecutionStatus.html" title="">JobExecutionStatus</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="JobSubmitter.html" title="Handle via which a "run" function passed to a ComplexFutureAction can submit jobs for execution." class="trait"></a><a href="JobSubmitter.html" title="Handle via which a "run" function passed to a ComplexFutureAction can submit jobs for execution.">JobSubmitter</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="NarrowDependency.html" title=":: DeveloperApi :: Base class for dependencies where each partition of the child RDD depends on a small number of partitions of the parent RDD." class="class"></a><a href="NarrowDependency.html" title=":: DeveloperApi :: Base class for dependencies where each partition of the child RDD depends on a small number of partitions of the parent RDD.">NarrowDependency</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="OneToOneDependency.html" title=":: DeveloperApi :: Represents a one-to-one dependency between partitions of the parent and child RDDs." class="class"></a><a href="OneToOneDependency.html" title=":: DeveloperApi :: Represents a one-to-one dependency between partitions of the parent and child RDDs.">OneToOneDependency</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="Partition.html" title="An identifier for a partition in an RDD." class="trait"></a><a href="Partition.html" title="An identifier for a partition in an RDD.">Partition</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="PartitionEvaluator.html" title="An evaluator for computing RDD partitions." class="trait"></a><a href="PartitionEvaluator.html" title="An evaluator for computing RDD partitions.">PartitionEvaluator</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="PartitionEvaluatorFactory.html" title="A factory to create PartitionEvaluator." class="trait"></a><a href="PartitionEvaluatorFactory.html" title="A factory to create PartitionEvaluator.">PartitionEvaluatorFactory</a></li><li class="current-entities indented3"><a href="Partitioner$.html" title="" class="object"></a> <a href="Partitioner.html" title="An object that defines how the elements in a key-value pair RDD are partitioned by key." class="class"></a><a href="Partitioner.html" title="An object that defines how the elements in a key-value pair RDD are partitioned by key.">Partitioner</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="QueryContext.html" title="Query context of a SparkThrowable." class="trait"></a><a href="QueryContext.html" title="Query context of a SparkThrowable.">QueryContext</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="QueryContextType.html" title="The type of QueryContext." class="class"></a><a href="QueryContextType.html" title="The type of QueryContext.">QueryContextType</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="RangeDependency.html" title=":: DeveloperApi :: Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs." class="class"></a><a href="RangeDependency.html" title=":: DeveloperApi :: Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs.">RangeDependency</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="RangePartitioner.html" title="A org.apache.spark.Partitioner that partitions sortable records by range into roughly equal ranges." class="class"></a><a href="RangePartitioner.html" title="A org.apache.spark.Partitioner that partitions sortable records by range into roughly equal ranges.">RangePartitioner</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="Resubmitted$.html" title=":: DeveloperApi :: A org.apache.spark.scheduler.ShuffleMapTask that completed successfully earlier, but we lost the executor before the stage completed." class="object"></a><a href="Resubmitted$.html" title=":: DeveloperApi :: A org.apache.spark.scheduler.ShuffleMapTask that completed successfully earlier, but we lost the executor before the stage completed.">Resubmitted</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SerializableWritable.html" title="" class="class"></a><a href="SerializableWritable.html" title="">SerializableWritable</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="ShuffleDependency.html" title=":: DeveloperApi :: Represents a dependency on the output of a shuffle stage." class="class"></a><a href="ShuffleDependency.html" title=":: DeveloperApi :: Represents a dependency on the output of a shuffle stage.">ShuffleDependency</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SimpleFutureAction.html" title="A FutureAction holding the result of an action that triggers a single job." class="class"></a><a href="SimpleFutureAction.html" title="A FutureAction holding the result of an action that triggers a single job.">SimpleFutureAction</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkConf.html" title="Configuration for a Spark application." class="class"></a><a href="SparkConf.html" title="Configuration for a Spark application.">SparkConf</a></li><li class="current-entities indented3"><a href="SparkContext$.html" title="The SparkContext object contains a number of implicit conversions and parameters for use with various Spark features." class="object"></a> <a href="SparkContext.html" title="Main entry point for Spark functionality." class="class"></a><a href="SparkContext.html" title="Main entry point for Spark functionality.">SparkContext</a></li><li class="current-entities indented3"><a href="SparkEnv$.html" title="" class="object"></a> <a href="SparkEnv.html" title=":: DeveloperApi :: Holds all the runtime environment objects for a running Spark instance (either master or worker), including the serializer, RpcEnv, block manager, map output tracker, etc." class="class"></a><a href="SparkEnv.html" title=":: DeveloperApi :: Holds all the runtime environment objects for a running Spark instance (either master or worker), including the serializer, RpcEnv, block manager, map output tracker, etc.">SparkEnv</a></li><li class="current-entities indented3"><a href="SparkException$.html" title="" class="object"></a> <a href="SparkException.html" title="" class="class"></a><a href="SparkException.html" title="">SparkException</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkExecutorInfo.html" title="Exposes information about Spark Executors." class="trait"></a><a href="SparkExecutorInfo.html" title="Exposes information about Spark Executors.">SparkExecutorInfo</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkFiles$.html" title="Resolves paths to files added through SparkContext.addFile()." class="object"></a><a href="SparkFiles$.html" title="Resolves paths to files added through SparkContext.addFile().">SparkFiles</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkFirehoseListener.html" title="Class that allows users to receive all SparkListener events." class="class"></a><a href="SparkFirehoseListener.html" title="Class that allows users to receive all SparkListener events.">SparkFirehoseListener</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkJobInfo.html" title="Exposes information about Spark Jobs." class="trait"></a><a href="SparkJobInfo.html" title="Exposes information about Spark Jobs.">SparkJobInfo</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkStageInfo.html" title="Exposes information about Spark Stages." class="trait"></a><a href="SparkStageInfo.html" title="Exposes information about Spark Stages.">SparkStageInfo</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkStatusTracker.html" title="Low-level status reporting APIs for monitoring job and stage progress." class="class"></a><a href="SparkStatusTracker.html" title="Low-level status reporting APIs for monitoring job and stage progress.">SparkStatusTracker</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="SparkThrowable.html" title="Interface mixed into Throwables thrown from Spark." class="trait"></a><a href="SparkThrowable.html" title="Interface mixed into Throwables thrown from Spark.">SparkThrowable</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="Success$.html" title=":: DeveloperApi :: Task succeeded." class="object"></a><a href="Success$.html" title=":: DeveloperApi :: Task succeeded.">Success</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="TaskCommitDenied.html" title=":: DeveloperApi :: Task requested the driver to commit, but was denied." class="class"></a><a href="TaskCommitDenied.html" title=":: DeveloperApi :: Task requested the driver to commit, but was denied.">TaskCommitDenied</a></li><li class="current-entities indented3"><a href="TaskContext$.html" title="" class="object"></a> <a href="TaskContext.html" title="Contextual information about a task which can be read or mutated during execution." class="class"></a><a href="TaskContext.html" title="Contextual information about a task which can be read or mutated during execution.">TaskContext</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="TaskEndReason.html" title=":: DeveloperApi :: Various possible reasons why a task ended." class="trait"></a><a href="TaskEndReason.html" title=":: DeveloperApi :: Various possible reasons why a task ended.">TaskEndReason</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="TaskFailedReason.html" title=":: DeveloperApi :: Various possible reasons why a task failed." class="trait"></a><a href="TaskFailedReason.html" title=":: DeveloperApi :: Various possible reasons why a task failed.">TaskFailedReason</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="TaskKilled.html" title=":: DeveloperApi :: Task was killed intentionally and needs to be rescheduled." class="class"></a><a href="TaskKilled.html" title=":: DeveloperApi :: Task was killed intentionally and needs to be rescheduled.">TaskKilled</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="TaskKilledException.html" title=":: DeveloperApi :: Exception thrown when a task is explicitly killed (i.e., task failure is expected)." class="class"></a><a href="TaskKilledException.html" title=":: DeveloperApi :: Exception thrown when a task is explicitly killed (i.e., task failure is expected).">TaskKilledException</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="TaskResultLost$.html" title=":: DeveloperApi :: The task finished successfully, but the result was lost from the executor's block manager before it was fetched." class="object"></a><a href="TaskResultLost$.html" title=":: DeveloperApi :: The task finished successfully, but the result was lost from the executor's block manager before it was fetched.">TaskResultLost</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="UnknownReason$.html" title=":: DeveloperApi :: We don't know why the task ended -- for example, because of a ClassNotFound exception when deserializing the task result." class="object"></a><a href="UnknownReason$.html" title=":: DeveloperApi :: We don't know why the task ended -- for example, because of a ClassNotFound exception when deserializing the task result.">UnknownReason</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="WritableConverter$.html" title="" class="object"></a><a href="WritableConverter$.html" title="">WritableConverter</a></li><li class="current-entities indented3"><span class="separator"></span> <a href="WritableFactory$.html" title="" class="object"></a><a href="WritableFactory$.html" title="">WritableFactory</a></li></ul></div></div><div id="content"><body class="package value"><div id="definition"><div class="big-circle package">p</div><p id="owner"><a href="../../index.html" name="org" id="org" class="extype">org</a>.<a href="../index.html" name="org.apache" id="org.apache" class="extype">apache</a></p><h1>spark<span class="permalink"><a href="../../../org/apache/spark/index.html" title="Permalink"><i class="material-icons"></i></a></span></h1></div><h4 id="signature" class="signature"><span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><span class="name">spark</span></span></h4><div id="comment" class="fullcommenttop"><div class="comment cmt"><p>Core Spark functionality. <a href="SparkContext.html" name="org.apache.spark.SparkContext" id="org.apache.spark.SparkContext" class="extype">org.apache.spark.SparkContext</a> serves as the main entry point to |
| Spark, while <a href="rdd/RDD.html" name="org.apache.spark.rdd.RDD" id="org.apache.spark.rdd.RDD" class="extype">org.apache.spark.rdd.RDD</a> is the data type representing a distributed collection, |
| and provides most parallel operations.</p><p>In addition, <a href="rdd/PairRDDFunctions.html" name="org.apache.spark.rdd.PairRDDFunctions" id="org.apache.spark.rdd.PairRDDFunctions" class="extype">org.apache.spark.rdd.PairRDDFunctions</a> contains operations available only on RDDs |
| of key-value pairs, such as <code>groupByKey</code> and <code>join</code>; <a href="rdd/DoubleRDDFunctions.html" name="org.apache.spark.rdd.DoubleRDDFunctions" id="org.apache.spark.rdd.DoubleRDDFunctions" class="extype">org.apache.spark.rdd.DoubleRDDFunctions</a> |
| contains operations available only on RDDs of Doubles; and |
| <a href="rdd/SequenceFileRDDFunctions.html" name="org.apache.spark.rdd.SequenceFileRDDFunctions" id="org.apache.spark.rdd.SequenceFileRDDFunctions" class="extype">org.apache.spark.rdd.SequenceFileRDDFunctions</a> contains operations available on RDDs that can |
| be saved as SequenceFiles. These operations are automatically available on any RDD of the right |
| type (e.g. RDD[(Int, Int)] through implicit conversions.</p><p>Java programmers should reference the <a href="api/java/index.html" name="org.apache.spark.api.java" id="org.apache.spark.api.java" class="extype">org.apache.spark.api.java</a> package |
| for Spark programming APIs in Java.</p><p>Classes and methods marked with <span class="experimental badge" style="float: none;"> |
| Experimental</span> are user-facing features which have not been officially adopted by the |
| Spark project. These are subject to change or removal in minor releases.</p><p>Classes and methods marked with <span class="developer badge" style="float: none;"> |
| Developer API</span> are intended for advanced users want to extend Spark through lower |
| level interfaces. These are subject to changes or removal in minor releases. |
| </p></div><dl class="attributes block"><dt>Source</dt><dd><a href="https://github.com/apache/spark/tree/v4.0.0-preview1/core/src/main/scala/org/apache/spark/package.scala" target="_blank">package.scala</a></dd></dl><div class="toggleContainer"><div class="toggle block"><span>Linear Supertypes</span><div class="superTypes hiddenContent"><span name="scala.AnyRef" class="extype">AnyRef</span>, <span name="scala.Any" class="extype">Any</span></div></div></div></div><div id="mbrsel"><div class="toggle"></div><div id="memberfilter"><i class="material-icons arrow"></i><span class="input"><input placeholder="Filter all members" id="mbrsel-input" type="text" accesskey="/"/></span><i class="clear material-icons"></i></div><div id="filterby"><div id="order"><span class="filtertype">Ordering</span><ol><li class="alpha in"><span>Alphabetic</span></li><li class="inherit out"><span>By Inheritance</span></li></ol></div><div class="ancestors"><span class="filtertype">Inherited<br/></span><ol id="linearization"><li class="in" name="org.apache.spark"><span>spark</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li></ol></div><div class="ancestors"><span class="filtertype"></span><ol><li class="hideall out"><span>Hide All</span></li><li class="showall in"><span>Show All</span></li></ol></div><div id="visbl"><span class="filtertype">Visibility</span><ol><li class="public in"><span>Public</span></li><li class="protected out"><span>Protected</span></li></ol></div></div></div><div id="template"><div id="allMembers"><div id="packages" class="package members"><h3>Package Members</h3><ol><li class="indented0 " name="org.apache.spark.api" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="api" class="anchorToMember"></a><a id="api:api" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/api/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="api/index.html" title=""><span class="name">api</span></a></span></li><li class="indented0 " name="org.apache.spark.broadcast" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="broadcast" class="anchorToMember"></a><a id="broadcast:broadcast" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/broadcast/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="broadcast/index.html" title="Spark's broadcast variables, used to broadcast immutable datasets to all nodes."><span class="name">broadcast</span></a></span><p class="shortcomment cmt">Spark's broadcast variables, used to broadcast immutable datasets to all nodes.</p></li><li class="indented0 " name="org.apache.spark.graphx" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="graphx" class="anchorToMember"></a><a id="graphx:graphx" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/graphx/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="graphx/index.html" title="ALPHA COMPONENT GraphX is a graph processing framework built on top of Spark."><span class="name">graphx</span></a></span><p class="shortcomment cmt"><span class="badge" style="float: right;">ALPHA COMPONENT</span> |
| GraphX is a graph processing framework built on top of Spark.</p></li><li class="indented0 " name="org.apache.spark.input" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="input" class="anchorToMember"></a><a id="input:input" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/input/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="input/index.html" title=""><span class="name">input</span></a></span></li><li class="indented0 " name="org.apache.spark.io" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="io" class="anchorToMember"></a><a id="io:io" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/io/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="io/index.html" title="IO codecs used for compression."><span class="name">io</span></a></span><p class="shortcomment cmt">IO codecs used for compression.</p><div class="fullcomment"><div class="comment cmt"><p>IO codecs used for compression. See <a href="io/CompressionCodec.html" name="org.apache.spark.io.CompressionCodec" id="org.apache.spark.io.CompressionCodec" class="extype">org.apache.spark.io.CompressionCodec</a>. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.launcher" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="launcher" class="anchorToMember"></a><a id="launcher:launcher" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/launcher/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="launcher/index.html" title=""><span class="name">launcher</span></a></span></li><li class="indented0 " name="org.apache.spark.mapred" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="mapred" class="anchorToMember"></a><a id="mapred:mapred" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/mapred/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="mapred/index.html" title=""><span class="name">mapred</span></a></span></li><li class="indented0 " name="org.apache.spark.metrics" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="metrics" class="anchorToMember"></a><a id="metrics:metrics" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/metrics/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="metrics/index.html" title=""><span class="name">metrics</span></a></span></li><li class="indented0 " name="org.apache.spark.ml" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="ml" class="anchorToMember"></a><a id="ml:ml" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ml/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="ml/index.html" title="DataFrame-based machine learning APIs to let users quickly assemble and configure practical machine learning pipelines."><span class="name">ml</span></a></span><p class="shortcomment cmt">DataFrame-based machine learning APIs to let users quickly assemble and configure practical |
| machine learning pipelines.</p></li><li class="indented0 " name="org.apache.spark.mllib" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="mllib" class="anchorToMember"></a><a id="mllib:mllib" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/mllib/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="mllib/index.html" title="RDD-based machine learning APIs (in maintenance mode)."><span class="name">mllib</span></a></span><p class="shortcomment cmt">RDD-based machine learning APIs (in maintenance mode).</p><div class="fullcomment"><div class="comment cmt"><p>RDD-based machine learning APIs (in maintenance mode).</p><p>The <code>spark.mllib</code> package is in maintenance mode as of the Spark 2.0.0 release to encourage |
| migration to the DataFrame-based APIs under the <a href="ml/index.html" name="org.apache.spark.ml" id="org.apache.spark.ml" class="extype">org.apache.spark.ml</a> package. |
| While in maintenance mode,</p><ul><li>no new features in the RDD-based <code>spark.mllib</code> package will be accepted, unless they block |
| implementing new features in the DataFrame-based <code>spark.ml</code> package;</li><li>bug fixes in the RDD-based APIs will still be accepted.</li></ul><p>The developers will continue adding more features to the DataFrame-based APIs in the 2.x series |
| to reach feature parity with the RDD-based APIs. |
| And once we reach feature parity, this package will be deprecated. |
| </p></div><dl class="attributes block"><dt>See also</dt><dd><span class="cmt"><p><a href="https://issues.apache.org/jira/browse/SPARK-4591">SPARK-4591</a> to track |
| the progress of feature parity</p></span></dd></dl></div></li><li class="indented0 " name="org.apache.spark.partial" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="partial" class="anchorToMember"></a><a id="partial:partial" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/partial/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="partial/index.html" title="Support for approximate results."><span class="name">partial</span></a></span><p class="shortcomment cmt">Support for approximate results.</p><div class="fullcomment"><div class="comment cmt"><p>Support for approximate results. This provides convenient api and also implementation for |
| approximate calculation. |
| </p></div><dl class="attributes block"><dt>See also</dt><dd><span class="cmt"><p><a href="rdd/RDD.html#countApprox(timeout:Long,confidence:Double):org.apache.spark.partial.PartialResult[org.apache.spark.partial.BoundedDouble]" name="org.apache.spark.rdd.RDD#countApprox" id="org.apache.spark.rdd.RDD#countApprox" class="extmbr">org.apache.spark.rdd.RDD.countApprox</a></p></span></dd></dl></div></li><li class="indented0 " name="org.apache.spark.paths" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="paths" class="anchorToMember"></a><a id="paths:paths" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/paths/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="paths/index.html" title=""><span class="name">paths</span></a></span></li><li class="indented0 " name="org.apache.spark.rdd" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="rdd" class="anchorToMember"></a><a id="rdd:rdd" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/rdd/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="rdd/index.html" title="Provides several RDD implementations."><span class="name">rdd</span></a></span><p class="shortcomment cmt">Provides several RDD implementations.</p><div class="fullcomment"><div class="comment cmt"><p>Provides several RDD implementations. See <a href="rdd/RDD.html" name="org.apache.spark.rdd.RDD" id="org.apache.spark.rdd.RDD" class="extype">org.apache.spark.rdd.RDD</a>. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.resource" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="resource" class="anchorToMember"></a><a id="resource:resource" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/resource/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="resource/index.html" title=""><span class="name">resource</span></a></span></li><li class="indented0 " name="org.apache.spark.scheduler" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="scheduler" class="anchorToMember"></a><a id="scheduler:scheduler" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/scheduler/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="scheduler/index.html" title="Spark's scheduling components."><span class="name">scheduler</span></a></span><p class="shortcomment cmt">Spark's scheduling components.</p><div class="fullcomment"><div class="comment cmt"><p>Spark's scheduling components. This includes the <code>org.apache.spark.scheduler.DAGScheduler</code> and |
| lower level <code>org.apache.spark.scheduler.TaskScheduler</code>. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.security" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="security" class="anchorToMember"></a><a id="security:security" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/security/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="security/index.html" title=""><span class="name">security</span></a></span></li><li class="indented0 " name="org.apache.spark.serializer" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="serializer" class="anchorToMember"></a><a id="serializer:serializer" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/serializer/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="serializer/index.html" title="Pluggable serializers for RDD and shuffle data."><span class="name">serializer</span></a></span><p class="shortcomment cmt">Pluggable serializers for RDD and shuffle data.</p><div class="fullcomment"><div class="comment cmt"><p>Pluggable serializers for RDD and shuffle data. |
| </p></div><dl class="attributes block"><dt>See also</dt><dd><span class="cmt"><p><a href="serializer/Serializer.html" name="org.apache.spark.serializer.Serializer" id="org.apache.spark.serializer.Serializer" class="extype">org.apache.spark.serializer.Serializer</a></p></span></dd></dl></div></li><li class="indented0 " name="org.apache.spark.shuffle" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="shuffle" class="anchorToMember"></a><a id="shuffle:shuffle" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/shuffle/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="shuffle/index.html" title=""><span class="name">shuffle</span></a></span></li><li class="indented0 " name="org.apache.spark.sql" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="sql" class="anchorToMember"></a><a id="sql:sql" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/sql/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="sql/index.html" title="Allows the execution of relational queries, including those expressed in SQL using Spark."><span class="name">sql</span></a></span><p class="shortcomment cmt">Allows the execution of relational queries, including those expressed in SQL using Spark.</p></li><li class="indented0 " name="org.apache.spark.status" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="status" class="anchorToMember"></a><a id="status:status" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/status/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="status/index.html" title=""><span class="name">status</span></a></span></li><li class="indented0 " name="org.apache.spark.storage" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="storage" class="anchorToMember"></a><a id="storage:storage" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/storage/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="storage/index.html" title=""><span class="name">storage</span></a></span></li><li class="indented0 " name="org.apache.spark.streaming" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="streaming" class="anchorToMember"></a><a id="streaming:streaming" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/streaming/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="streaming/index.html" title="Spark Streaming functionality."><span class="name">streaming</span></a></span><p class="shortcomment cmt">Spark Streaming functionality.</p><div class="fullcomment"><div class="comment cmt"><p>Spark Streaming functionality. <a href="streaming/StreamingContext.html" name="org.apache.spark.streaming.StreamingContext" id="org.apache.spark.streaming.StreamingContext" class="extype">org.apache.spark.streaming.StreamingContext</a> serves as the main |
| entry point to Spark Streaming, while <a href="streaming/dstream/DStream.html" name="org.apache.spark.streaming.dstream.DStream" id="org.apache.spark.streaming.dstream.DStream" class="extype">org.apache.spark.streaming.dstream.DStream</a> is the data |
| type representing a continuous sequence of RDDs, representing a continuous stream of data.</p><p>In addition, <a href="streaming/dstream/PairDStreamFunctions.html" name="org.apache.spark.streaming.dstream.PairDStreamFunctions" id="org.apache.spark.streaming.dstream.PairDStreamFunctions" class="extype">org.apache.spark.streaming.dstream.PairDStreamFunctions</a> contains operations |
| available only on DStreams |
| of key-value pairs, such as <code>groupByKey</code> and <code>reduceByKey</code>. These operations are automatically |
| available on any DStream of the right type (e.g. DStream[(Int, Int)] through implicit |
| conversions.</p><p>For the Java API of Spark Streaming, take a look at the |
| <a href="streaming/api/java/JavaStreamingContext.html" name="org.apache.spark.streaming.api.java.JavaStreamingContext" id="org.apache.spark.streaming.api.java.JavaStreamingContext" class="extype">org.apache.spark.streaming.api.java.JavaStreamingContext</a> which serves as the entry point, and |
| the <a href="streaming/api/java/JavaDStream.html" name="org.apache.spark.streaming.api.java.JavaDStream" id="org.apache.spark.streaming.api.java.JavaDStream" class="extype">org.apache.spark.streaming.api.java.JavaDStream</a> and the |
| <a href="streaming/api/java/JavaPairDStream.html" name="org.apache.spark.streaming.api.java.JavaPairDStream" id="org.apache.spark.streaming.api.java.JavaPairDStream" class="extype">org.apache.spark.streaming.api.java.JavaPairDStream</a> which have the DStream functionality. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.types" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="types" class="anchorToMember"></a><a id="types:types" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/types/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="types/index.html" title=""><span class="name">types</span></a></span></li><li class="indented0 " name="org.apache.spark.ui" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="ui" class="anchorToMember"></a><a id="ui:ui" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ui/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="ui/index.html" title=""><span class="name">ui</span></a></span></li><li class="indented0 " name="org.apache.spark.unsafe" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="unsafe" class="anchorToMember"></a><a id="unsafe:unsafe" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/unsafe/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="unsafe/index.html" title=""><span class="name">unsafe</span></a></span></li><li class="indented0 " name="org.apache.spark.util" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="util" class="anchorToMember"></a><a id="util:util" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/util/index.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">package</span></span> <span class="symbol"><a href="util/index.html" title="Spark utilities."><span class="name">util</span></a></span><p class="shortcomment cmt">Spark utilities.</p></li></ol></div><div id="types" class="types members"><h3>Type Members</h3><ol><li class="indented0 " name="org.apache.spark.Aggregator" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="Aggregator[K,V,C]extendsProductwithSerializable" class="anchorToMember"></a><a id="Aggregator[K,V,C]:Aggregator[K,V,C]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/Aggregator.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case class</span></span> <span class="symbol"><a href="Aggregator.html" title=":: DeveloperApi :: A set of functions used to aggregate data."><span class="name">Aggregator</span></a><span class="tparams">[<span name="K">K</span>, <span name="V">V</span>, <span name="C">C</span>]</span><span class="params">(<span name="createCombiner">createCombiner: (<span name="org.apache.spark.Aggregator.V" class="extype">V</span>) => <span name="org.apache.spark.Aggregator.C" class="extype">C</span></span>, <span name="mergeValue">mergeValue: (<span name="org.apache.spark.Aggregator.C" class="extype">C</span>, <span name="org.apache.spark.Aggregator.V" class="extype">V</span>) => <span name="org.apache.spark.Aggregator.C" class="extype">C</span></span>, <span name="mergeCombiners">mergeCombiners: (<span name="org.apache.spark.Aggregator.C" class="extype">C</span>, <span name="org.apache.spark.Aggregator.C" class="extype">C</span>) => <span name="org.apache.spark.Aggregator.C" class="extype">C</span></span>)</span><span class="result"> extends <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| A set of functions used to aggregate data.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| A set of functions used to aggregate data. |
| </p></div><dl class="paramcmts block"><dt class="param">createCombiner</dt><dd class="cmt"><p>function to create the initial value of the aggregation.</p></dd><dt class="param">mergeValue</dt><dd class="cmt"><p>function to merge a new value into the aggregation result.</p></dd><dt class="param">mergeCombiners</dt><dd class="cmt"><p>function to merge outputs from multiple mergeValue function.</p></dd></dl><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.BarrierTaskContext" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="BarrierTaskContextextendsTaskContextwithLogging" class="anchorToMember"></a><a id="BarrierTaskContext:BarrierTaskContext" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/BarrierTaskContext.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="BarrierTaskContext.html" title=":: Experimental :: A TaskContext with extra contextual info and tooling for tasks in a barrier stage."><span class="name">BarrierTaskContext</span></a><span class="result"> extends <a href="TaskContext.html" name="org.apache.spark.TaskContext" id="org.apache.spark.TaskContext" class="extype">TaskContext</a> with <span name="org.apache.spark.internal.Logging" class="extype">Logging</span></span></span><p class="shortcomment cmt">:: Experimental :: |
| A <a href="TaskContext.html" name="org.apache.spark.TaskContext" id="org.apache.spark.TaskContext" class="extype">TaskContext</a> with extra contextual info and tooling for tasks in a barrier stage.</p><div class="fullcomment"><div class="comment cmt"><p>:: Experimental :: |
| A <a href="TaskContext.html" name="org.apache.spark.TaskContext" id="org.apache.spark.TaskContext" class="extype">TaskContext</a> with extra contextual info and tooling for tasks in a barrier stage. |
| Use <a href="BarrierTaskContext$.html#get():org.apache.spark.BarrierTaskContext" name="org.apache.spark.BarrierTaskContext#get" id="org.apache.spark.BarrierTaskContext#get" class="extmbr">BarrierTaskContext#get</a> to obtain the barrier context for a running barrier task. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@Experimental</span><span class="args">()</span> <span class="name">@Since</span><span class="args">(<span><span class="symbol">"2.4.0"</span></span>)</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.BarrierTaskInfo" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="BarrierTaskInfoextendsAnyRef" class="anchorToMember"></a><a id="BarrierTaskInfo:BarrierTaskInfo" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/BarrierTaskInfo.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="BarrierTaskInfo.html" title=":: Experimental :: Carries all task infos of a barrier task."><span class="name">BarrierTaskInfo</span></a><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">:: Experimental :: |
| Carries all task infos of a barrier task.</p><div class="fullcomment"><div class="comment cmt"><p>:: Experimental :: |
| Carries all task infos of a barrier task. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@Experimental</span><span class="args">()</span> <span class="name">@Since</span><span class="args">(<span><span class="symbol">"2.4.0"</span></span>)</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.ComplexFutureAction" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="ComplexFutureAction[T]extendsFutureAction[T]" class="anchorToMember"></a><a id="ComplexFutureAction[T]:ComplexFutureAction[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ComplexFutureAction.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="ComplexFutureAction.html" title="A FutureAction for actions that could trigger multiple Spark jobs."><span class="name">ComplexFutureAction</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <a href="FutureAction.html" name="org.apache.spark.FutureAction" id="org.apache.spark.FutureAction" class="extype">FutureAction</a>[<span name="org.apache.spark.ComplexFutureAction.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">A <a href="FutureAction.html" name="org.apache.spark.FutureAction" id="org.apache.spark.FutureAction" class="extype">FutureAction</a> for actions that could trigger multiple Spark jobs.</p><div class="fullcomment"><div class="comment cmt"><p>A <a href="FutureAction.html" name="org.apache.spark.FutureAction" id="org.apache.spark.FutureAction" class="extype">FutureAction</a> for actions that could trigger multiple Spark jobs. Examples include take, |
| takeSample. Cancellation works by setting the cancelled flag to true and cancelling any pending |
| jobs. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.Dependency" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="Dependency[T]extendsSerializable" class="anchorToMember"></a><a id="Dependency[T]:Dependency[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/Dependency.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">abstract </span> <span class="kind">class</span></span> <span class="symbol"><a href="Dependency.html" title=":: DeveloperApi :: Base class for dependencies."><span class="name">Dependency</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Base class for dependencies.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Base class for dependencies. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.ErrorClassesJsonReader" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="ErrorClassesJsonReaderextendsAnyRef" class="anchorToMember"></a><a id="ErrorClassesJsonReader:ErrorClassesJsonReader" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ErrorClassesJsonReader.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="ErrorClassesJsonReader.html" title="A reader to load error information from one or more JSON files."><span class="name">ErrorClassesJsonReader</span></a><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">A reader to load error information from one or more JSON files.</p><div class="fullcomment"><div class="comment cmt"><p>A reader to load error information from one or more JSON files. Note that, if one error appears |
| in more than one JSON files, the latter wins. |
| Please read common/utils/src/main/resources/error/README.md for more details. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.ExceptionFailure" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="ExceptionFailureextendsTaskFailedReasonwithProductwithSerializable" class="anchorToMember"></a><a id="ExceptionFailure:ExceptionFailure" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ExceptionFailure.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case class</span></span> <span class="symbol"><a href="ExceptionFailure.html" title=":: DeveloperApi :: Task failed due to a runtime exception."><span class="name">ExceptionFailure</span></a><span class="params">(<span name="className">className: <span name="scala.Predef.String" class="extype">String</span></span>, <span name="description">description: <span name="scala.Predef.String" class="extype">String</span></span>, <span name="stackTrace">stackTrace: <span name="scala.Array" class="extype">Array</span>[<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/StackTraceElement.html#java.lang.StackTraceElement" name="java.lang.StackTraceElement" id="java.lang.StackTraceElement" class="extype">StackTraceElement</a>]</span>, <span name="fullStackTrace">fullStackTrace: <span name="scala.Predef.String" class="extype">String</span></span>, <span name="exceptionWrapper">exceptionWrapper: <span name="scala.Option" class="extype">Option</span>[<span name="org.apache.spark.ThrowableSerializationWrapper" class="extype">ThrowableSerializationWrapper</span>]</span>, <span name="accumUpdates">accumUpdates: <span name="scala.Seq" class="extype">Seq</span>[<a href="scheduler/AccumulableInfo.html" name="org.apache.spark.scheduler.AccumulableInfo" id="org.apache.spark.scheduler.AccumulableInfo" class="extype">AccumulableInfo</a>] = <span class="symbol">Seq.empty</span></span>, <span name="accums">accums: <span name="scala.Seq" class="extype">Seq</span>[<a href="util/AccumulatorV2.html" name="org.apache.spark.util.AccumulatorV2" id="org.apache.spark.util.AccumulatorV2" class="extype">AccumulatorV2</a>[_, _]] = <span class="symbol">Nil</span></span>, <span name="metricPeaks">metricPeaks: <span name="scala.Seq" class="extype">Seq</span>[<span name="scala.Long" class="extype">Long</span>] = <span class="symbol">Seq.empty</span></span>)</span><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Task failed due to a runtime exception.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Task failed due to a runtime exception. This is the most common failure case and also captures |
| user program exceptions.</p><p><code>stackTrace</code> contains the stack trace of the exception itself. It still exists for backward |
| compatibility. It's better to use <code>this(e: Throwable, metrics: Option[TaskMetrics])</code> to |
| create <code>ExceptionFailure</code> as it will handle the backward compatibility properly.</p><p><code>fullStackTrace</code> is a better representation of the stack trace because it contains the whole |
| stack trace including the exception and its causes</p><p><code>exception</code> is the actual exception that caused the task to fail. It may be <code>None</code> in |
| the case that the exception is not in fact serializable. If a task fails more than |
| once (due to retries), <code>exception</code> is that one that caused the last failure. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.ExecutorLostFailure" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="ExecutorLostFailureextendsTaskFailedReasonwithProductwithSerializable" class="anchorToMember"></a><a id="ExecutorLostFailure:ExecutorLostFailure" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ExecutorLostFailure.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case class</span></span> <span class="symbol"><a href="ExecutorLostFailure.html" title=":: DeveloperApi :: The task failed because the executor that it was running on was lost."><span class="name">ExecutorLostFailure</span></a><span class="params">(<span name="execId">execId: <span name="scala.Predef.String" class="extype">String</span></span>, <span name="exitCausedByApp">exitCausedByApp: <span name="scala.Boolean" class="extype">Boolean</span> = <span class="symbol">true</span></span>, <span name="reason">reason: <span name="scala.Option" class="extype">Option</span>[<span name="scala.Predef.String" class="extype">String</span>]</span>)</span><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| The task failed because the executor that it was running on was lost.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| The task failed because the executor that it was running on was lost. This may happen because |
| the task crashed the JVM. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.FetchFailed" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="FetchFailedextendsTaskFailedReasonwithProductwithSerializable" class="anchorToMember"></a><a id="FetchFailed:FetchFailed" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/FetchFailed.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case class</span></span> <span class="symbol"><a href="FetchFailed.html" title=":: DeveloperApi :: Task failed to fetch shuffle data from a remote node."><span class="name">FetchFailed</span></a><span class="params">(<span name="bmAddress">bmAddress: <a href="storage/BlockManagerId.html" name="org.apache.spark.storage.BlockManagerId" id="org.apache.spark.storage.BlockManagerId" class="extype">BlockManagerId</a></span>, <span name="shuffleId">shuffleId: <span name="scala.Int" class="extype">Int</span></span>, <span name="mapId">mapId: <span name="scala.Long" class="extype">Long</span></span>, <span name="mapIndex">mapIndex: <span name="scala.Int" class="extype">Int</span></span>, <span name="reduceId">reduceId: <span name="scala.Int" class="extype">Int</span></span>, <span name="message">message: <span name="scala.Predef.String" class="extype">String</span></span>)</span><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Task failed to fetch shuffle data from a remote node.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Task failed to fetch shuffle data from a remote node. Probably means we have lost the remote |
| executors the task is trying to fetch from, and thus need to rerun the previous stage. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.FutureAction" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="FutureAction[T]extendsFuture[T]" class="anchorToMember"></a><a id="FutureAction[T]:FutureAction[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/FutureAction.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="FutureAction.html" title="A future for the result of an action to support cancellation."><span class="name">FutureAction</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span name="scala.concurrent.Future" class="extype">Future</span>[<span name="org.apache.spark.FutureAction.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">A future for the result of an action to support cancellation.</p><div class="fullcomment"><div class="comment cmt"><p>A future for the result of an action to support cancellation. This is an extension of the |
| Scala Future interface to support cancellation. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.HashPartitioner" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="HashPartitionerextendsPartitioner" class="anchorToMember"></a><a id="HashPartitioner:HashPartitioner" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/HashPartitioner.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="HashPartitioner.html" title="A org.apache.spark.Partitioner that implements hash-based partitioning using Java's Object.hashCode."><span class="name">HashPartitioner</span></a><span class="result"> extends <a href="Partitioner.html" name="org.apache.spark.Partitioner" id="org.apache.spark.Partitioner" class="extype">Partitioner</a></span></span><p class="shortcomment cmt">A <a href="Partitioner.html" name="org.apache.spark.Partitioner" id="org.apache.spark.Partitioner" class="extype">org.apache.spark.Partitioner</a> that implements hash-based partitioning using |
| Java's <code>Object.hashCode</code>.</p><div class="fullcomment"><div class="comment cmt"><p>A <a href="Partitioner.html" name="org.apache.spark.Partitioner" id="org.apache.spark.Partitioner" class="extype">org.apache.spark.Partitioner</a> that implements hash-based partitioning using |
| Java's <code>Object.hashCode</code>.</p><p>Java arrays have hashCodes that are based on the arrays' identities rather than their contents, |
| so attempting to partition an RDD[Array[_]] or RDD[(Array[_], _)] using a HashPartitioner will |
| produce an unexpected or incorrect result. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.InterruptibleIterator" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="InterruptibleIterator[+T]extendsIterator[T]" class="anchorToMember"></a><a id="InterruptibleIterator[+T]:InterruptibleIterator[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/InterruptibleIterator.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="InterruptibleIterator.html" title=":: DeveloperApi :: An iterator that wraps around an existing iterator to provide task killing functionality."><span class="name">InterruptibleIterator</span></a><span class="tparams">[<span name="T">+T</span>]</span><span class="result"> extends <span name="scala.Iterator" class="extype">Iterator</span>[<span name="org.apache.spark.InterruptibleIterator.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| An iterator that wraps around an existing iterator to provide task killing functionality.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| An iterator that wraps around an existing iterator to provide task killing functionality. |
| It works by checking the interrupted flag in <a href="TaskContext.html" name="org.apache.spark.TaskContext" id="org.apache.spark.TaskContext" class="extype">TaskContext</a>. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.JobExecutionStatus" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="JobExecutionStatusextendsEnum[org.apache.spark.JobExecutionStatus]" class="anchorToMember"></a><a id="JobExecutionStatus:JobExecutionStatus" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/JobExecutionStatus.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">sealed final </span> <span class="kind">class</span></span> <span class="symbol"><a href="JobExecutionStatus.html" title=""><span class="name">JobExecutionStatus</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Enum.html#java.lang.Enum" name="java.lang.Enum" id="java.lang.Enum" class="extype">Enum</a>[<a href="JobExecutionStatus.html" name="org.apache.spark.JobExecutionStatus" id="org.apache.spark.JobExecutionStatus" class="extype">JobExecutionStatus</a>]</span></span></li><li class="indented0 " name="org.apache.spark.JobSubmitter" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="JobSubmitterextendsAnyRef" class="anchorToMember"></a><a id="JobSubmitter:JobSubmitter" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/JobSubmitter.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="JobSubmitter.html" title="Handle via which a "run" function passed to a ComplexFutureAction can submit jobs for execution."><span class="name">JobSubmitter</span></a><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">Handle via which a "run" function passed to a <a href="ComplexFutureAction.html" name="org.apache.spark.ComplexFutureAction" id="org.apache.spark.ComplexFutureAction" class="extype">ComplexFutureAction</a> |
| can submit jobs for execution.</p><div class="fullcomment"><div class="comment cmt"><p>Handle via which a "run" function passed to a <a href="ComplexFutureAction.html" name="org.apache.spark.ComplexFutureAction" id="org.apache.spark.ComplexFutureAction" class="extype">ComplexFutureAction</a> |
| can submit jobs for execution. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.NarrowDependency" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="NarrowDependency[T]extendsDependency[T]" class="anchorToMember"></a><a id="NarrowDependency[T]:NarrowDependency[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/NarrowDependency.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">abstract </span> <span class="kind">class</span></span> <span class="symbol"><a href="NarrowDependency.html" title=":: DeveloperApi :: Base class for dependencies where each partition of the child RDD depends on a small number of partitions of the parent RDD."><span class="name">NarrowDependency</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <a href="Dependency.html" name="org.apache.spark.Dependency" id="org.apache.spark.Dependency" class="extype">Dependency</a>[<span name="org.apache.spark.NarrowDependency.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Base class for dependencies where each partition of the child RDD depends on a small number |
| of partitions of the parent RDD.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Base class for dependencies where each partition of the child RDD depends on a small number |
| of partitions of the parent RDD. Narrow dependencies allow for pipelined execution. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.OneToOneDependency" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="OneToOneDependency[T]extendsNarrowDependency[T]" class="anchorToMember"></a><a id="OneToOneDependency[T]:OneToOneDependency[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/OneToOneDependency.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="OneToOneDependency.html" title=":: DeveloperApi :: Represents a one-to-one dependency between partitions of the parent and child RDDs."><span class="name">OneToOneDependency</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <a href="NarrowDependency.html" name="org.apache.spark.NarrowDependency" id="org.apache.spark.NarrowDependency" class="extype">NarrowDependency</a>[<span name="org.apache.spark.OneToOneDependency.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Represents a one-to-one dependency between partitions of the parent and child RDDs.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Represents a one-to-one dependency between partitions of the parent and child RDDs. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.Partition" group="Ungrouped" fullComment="no" data-isabs="true" visbl="pub"><a id="PartitionextendsSerializable" class="anchorToMember"></a><a id="Partition:Partition" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/Partition.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="Partition.html" title="An identifier for a partition in an RDD."><span class="name">Partition</span></a><span class="result"> extends <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">An identifier for a partition in an RDD.</p></li><li class="indented0 " name="org.apache.spark.PartitionEvaluator" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="PartitionEvaluator[T,U]extendsAnyRef" class="anchorToMember"></a><a id="PartitionEvaluator[T,U]:PartitionEvaluator[T,U]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/PartitionEvaluator.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="PartitionEvaluator.html" title="An evaluator for computing RDD partitions."><span class="name">PartitionEvaluator</span></a><span class="tparams">[<span name="T">T</span>, <span name="U">U</span>]</span><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">An evaluator for computing RDD partitions.</p><div class="fullcomment"><div class="comment cmt"><p>An evaluator for computing RDD partitions. Spark serializes and sends |
| <a href="PartitionEvaluatorFactory.html" name="org.apache.spark.PartitionEvaluatorFactory" id="org.apache.spark.PartitionEvaluatorFactory" class="extype">PartitionEvaluatorFactory</a> to executors, and then creates <a href="PartitionEvaluator.html" name="org.apache.spark.PartitionEvaluator" id="org.apache.spark.PartitionEvaluator" class="extype">PartitionEvaluator</a> via the |
| factory at the executor side. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> <span class="name">@Since</span><span class="args">(<span><span class="symbol">"3.5.0"</span></span>)</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.PartitionEvaluatorFactory" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="PartitionEvaluatorFactory[T,U]extendsSerializable" class="anchorToMember"></a><a id="PartitionEvaluatorFactory[T,U]:PartitionEvaluatorFactory[T,U]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/PartitionEvaluatorFactory.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="PartitionEvaluatorFactory.html" title="A factory to create PartitionEvaluator."><span class="name">PartitionEvaluatorFactory</span></a><span class="tparams">[<span name="T">T</span>, <span name="U">U</span>]</span><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span><p class="shortcomment cmt">A factory to create <a href="PartitionEvaluator.html" name="org.apache.spark.PartitionEvaluator" id="org.apache.spark.PartitionEvaluator" class="extype">PartitionEvaluator</a>.</p><div class="fullcomment"><div class="comment cmt"><p>A factory to create <a href="PartitionEvaluator.html" name="org.apache.spark.PartitionEvaluator" id="org.apache.spark.PartitionEvaluator" class="extype">PartitionEvaluator</a>. Spark serializes and sends |
| <a href="PartitionEvaluatorFactory.html" name="org.apache.spark.PartitionEvaluatorFactory" id="org.apache.spark.PartitionEvaluatorFactory" class="extype">PartitionEvaluatorFactory</a> to executors, and then creates <a href="PartitionEvaluator.html" name="org.apache.spark.PartitionEvaluator" id="org.apache.spark.PartitionEvaluator" class="extype">PartitionEvaluator</a> via the |
| factory at the executor side. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> <span class="name">@Since</span><span class="args">(<span><span class="symbol">"3.5.0"</span></span>)</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.Partitioner" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="PartitionerextendsSerializable" class="anchorToMember"></a><a id="Partitioner:Partitioner" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/Partitioner.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">abstract </span> <span class="kind">class</span></span> <span class="symbol"><a href="Partitioner.html" title="An object that defines how the elements in a key-value pair RDD are partitioned by key."><span class="name">Partitioner</span></a><span class="result"> extends <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">An object that defines how the elements in a key-value pair RDD are partitioned by key.</p><div class="fullcomment"><div class="comment cmt"><p>An object that defines how the elements in a key-value pair RDD are partitioned by key. |
| Maps each key to a partition ID, from 0 to <code>numPartitions - 1</code>.</p><p>Note that, partitioner must be deterministic, i.e. it must return the same partition id given |
| the same partition key. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.QueryContext" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="QueryContextextendsObject" class="anchorToMember"></a><a id="QueryContext:QueryContext" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/QueryContext.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="QueryContext.html" title="Query context of a SparkThrowable."><span class="name">QueryContext</span></a><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">Query context of a <code><a href="SparkThrowable.html" name="org.apache.spark.SparkThrowable" id="org.apache.spark.SparkThrowable" class="extype">SparkThrowable</a></code>.</p><div class="fullcomment"><div class="comment cmt"><p>Query context of a <code><a href="SparkThrowable.html" name="org.apache.spark.SparkThrowable" id="org.apache.spark.SparkThrowable" class="extype">SparkThrowable</a></code>. It helps users understand where error occur |
| while executing queries. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@Evolving</span><span class="args">()</span> </dd><dt>Since</dt><dd><p>3.4.0</p></dd></dl></div></li><li class="indented0 " name="org.apache.spark.QueryContextType" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="QueryContextTypeextendsEnum[org.apache.spark.QueryContextType]" class="anchorToMember"></a><a id="QueryContextType:QueryContextType" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/QueryContextType.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">sealed final </span> <span class="kind">class</span></span> <span class="symbol"><a href="QueryContextType.html" title="The type of QueryContext."><span class="name">QueryContextType</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Enum.html#java.lang.Enum" name="java.lang.Enum" id="java.lang.Enum" class="extype">Enum</a>[<a href="QueryContextType.html" name="org.apache.spark.QueryContextType" id="org.apache.spark.QueryContextType" class="extype">QueryContextType</a>]</span></span><p class="shortcomment cmt">The type of <code><a href="QueryContext.html" name="org.apache.spark.QueryContext" id="org.apache.spark.QueryContext" class="extype">QueryContext</a></code>.</p><div class="fullcomment"><div class="comment cmt"><p>The type of <code><a href="QueryContext.html" name="org.apache.spark.QueryContext" id="org.apache.spark.QueryContext" class="extype">QueryContext</a></code>. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@Evolving</span><span class="args">()</span> </dd><dt>Since</dt><dd><p>4.0.0</p></dd></dl></div></li><li class="indented0 " name="org.apache.spark.RangeDependency" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="RangeDependency[T]extendsNarrowDependency[T]" class="anchorToMember"></a><a id="RangeDependency[T]:RangeDependency[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/RangeDependency.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="RangeDependency.html" title=":: DeveloperApi :: Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs."><span class="name">RangeDependency</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <a href="NarrowDependency.html" name="org.apache.spark.NarrowDependency" id="org.apache.spark.NarrowDependency" class="extype">NarrowDependency</a>[<span name="org.apache.spark.RangeDependency.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs.</p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.RangePartitioner" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="RangePartitioner[K,V]extendsPartitioner" class="anchorToMember"></a><a id="RangePartitioner[K,V]:RangePartitioner[K,V]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/RangePartitioner.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="RangePartitioner.html" title="A org.apache.spark.Partitioner that partitions sortable records by range into roughly equal ranges."><span class="name">RangePartitioner</span></a><span class="tparams">[<span name="K">K</span>, <span name="V">V</span>]</span><span class="result"> extends <a href="Partitioner.html" name="org.apache.spark.Partitioner" id="org.apache.spark.Partitioner" class="extype">Partitioner</a></span></span><p class="shortcomment cmt">A <a href="Partitioner.html" name="org.apache.spark.Partitioner" id="org.apache.spark.Partitioner" class="extype">org.apache.spark.Partitioner</a> that partitions sortable records by range into roughly |
| equal ranges.</p><div class="fullcomment"><div class="comment cmt"><p>A <a href="Partitioner.html" name="org.apache.spark.Partitioner" id="org.apache.spark.Partitioner" class="extype">org.apache.spark.Partitioner</a> that partitions sortable records by range into roughly |
| equal ranges. The ranges are determined by sampling the content of the RDD passed in. |
| </p></div><dl class="attributes block"><dt>Note</dt><dd><span class="cmt"><p>The actual number of partitions created by the RangePartitioner might not be the same |
| as the <code>partitions</code> parameter, in the case where the number of sampled records is less than |
| the value of <code>partitions</code>.</p></span></dd></dl></div></li><li class="indented0 " name="org.apache.spark.SerializableWritable" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="SerializableWritable[T<:org.apache.hadoop.io.Writable]extendsSerializable" class="anchorToMember"></a><a id="SerializableWritable[T<:Writable]:SerializableWritable[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SerializableWritable.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SerializableWritable.html" title=""><span class="name">SerializableWritable</span></a><span class="tparams">[<span name="T">T <: <span name="org.apache.hadoop.io.Writable" class="extype">Writable</span></span>]</span><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span><div class="fullcomment"><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.ShuffleDependency" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="ShuffleDependency[K,V,C]extendsDependency[Product2[K,V]]withLogging" class="anchorToMember"></a><a id="ShuffleDependency[K,V,C]:ShuffleDependency[K,V,C]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ShuffleDependency.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="ShuffleDependency.html" title=":: DeveloperApi :: Represents a dependency on the output of a shuffle stage."><span class="name">ShuffleDependency</span></a><span class="tparams">[<span name="K">K</span>, <span name="V">V</span>, <span name="C">C</span>]</span><span class="result"> extends <a href="Dependency.html" name="org.apache.spark.Dependency" id="org.apache.spark.Dependency" class="extype">Dependency</a>[<span name="scala.Product2" class="extype">Product2</span>[<span name="org.apache.spark.ShuffleDependency.K" class="extype">K</span>, <span name="org.apache.spark.ShuffleDependency.V" class="extype">V</span>]] with <span name="org.apache.spark.internal.Logging" class="extype">Logging</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Represents a dependency on the output of a shuffle stage.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Represents a dependency on the output of a shuffle stage. Note that in the case of shuffle, |
| the RDD is transient since we don't need it on the executor side. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.SimpleFutureAction" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="SimpleFutureAction[T]extendsFutureAction[T]" class="anchorToMember"></a><a id="SimpleFutureAction[T]:SimpleFutureAction[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SimpleFutureAction.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SimpleFutureAction.html" title="A FutureAction holding the result of an action that triggers a single job."><span class="name">SimpleFutureAction</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <a href="FutureAction.html" name="org.apache.spark.FutureAction" id="org.apache.spark.FutureAction" class="extype">FutureAction</a>[<span name="org.apache.spark.SimpleFutureAction.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">A <a href="FutureAction.html" name="org.apache.spark.FutureAction" id="org.apache.spark.FutureAction" class="extype">FutureAction</a> holding the result of an action that triggers a single job.</p><div class="fullcomment"><div class="comment cmt"><p>A <a href="FutureAction.html" name="org.apache.spark.FutureAction" id="org.apache.spark.FutureAction" class="extype">FutureAction</a> holding the result of an action that triggers a single job. Examples include |
| count, collect, reduce. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.SparkConf" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="SparkConfextendsCloneablewithLoggingwithSerializable" class="anchorToMember"></a><a id="SparkConf:SparkConf" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkConf.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SparkConf.html" title="Configuration for a Spark application."><span class="name">SparkConf</span></a><span class="result"> extends <span name="scala.Cloneable" class="extype">Cloneable</span> with <span name="org.apache.spark.internal.Logging" class="extype">Logging</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">Configuration for a Spark application.</p><div class="fullcomment"><div class="comment cmt"><p>Configuration for a Spark application. Used to set various Spark parameters as key-value pairs.</p><p>Most of the time, you would create a SparkConf object with <code>new SparkConf()</code>, which will load |
| values from any <code>spark.*</code> Java system properties set in your application as well. In this case, |
| parameters you set directly on the <code>SparkConf</code> object take priority over system properties.</p><p>For unit tests, you can also call <code>new SparkConf(false)</code> to skip loading external settings and |
| get the same configuration no matter what the system properties are.</p><p>All setter methods in this class support chaining. For example, you can write |
| <code>new SparkConf().setMaster("local").setAppName("My app")</code>. |
| </p></div><dl class="attributes block"><dt>Note</dt><dd><span class="cmt"><p>Once a SparkConf object is passed to Spark, it is cloned and can no longer be modified |
| by the user. Spark does not support modifying the configuration at runtime.</p></span></dd></dl></div></li><li class="indented0 " name="org.apache.spark.SparkContext" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="SparkContextextendsLogging" class="anchorToMember"></a><a id="SparkContext:SparkContext" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkContext.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SparkContext.html" title="Main entry point for Spark functionality."><span class="name">SparkContext</span></a><span class="result"> extends <span name="org.apache.spark.internal.Logging" class="extype">Logging</span></span></span><p class="shortcomment cmt">Main entry point for Spark functionality.</p><div class="fullcomment"><div class="comment cmt"><p>Main entry point for Spark functionality. A SparkContext represents the connection to a Spark |
| cluster, and can be used to create RDDs, accumulators and broadcast variables on that cluster. |
| </p></div><dl class="attributes block"><dt>Note</dt><dd><span class="cmt"><p>Only one <code>SparkContext</code> should be active per JVM. You must <code>stop()</code> the |
| active <code>SparkContext</code> before creating a new one.</p></span></dd></dl></div></li><li class="indented0 " name="org.apache.spark.SparkEnv" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="SparkEnvextendsLogging" class="anchorToMember"></a><a id="SparkEnv:SparkEnv" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkEnv.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SparkEnv.html" title=":: DeveloperApi :: Holds all the runtime environment objects for a running Spark instance (either master or worker), including the serializer, RpcEnv, block manager, map output tracker, etc."><span class="name">SparkEnv</span></a><span class="result"> extends <span name="org.apache.spark.internal.Logging" class="extype">Logging</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Holds all the runtime environment objects for a running Spark instance (either master or worker), |
| including the serializer, RpcEnv, block manager, map output tracker, etc.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Holds all the runtime environment objects for a running Spark instance (either master or worker), |
| including the serializer, RpcEnv, block manager, map output tracker, etc. Currently |
| Spark code finds the SparkEnv through a global variable, so all the threads can access the same |
| SparkEnv. It can be accessed by SparkEnv.get (e.g. after creating a SparkContext). |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.SparkException" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SparkExceptionextendsExceptionwithSparkThrowable" class="anchorToMember"></a><a id="SparkException:SparkException" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkException.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SparkException.html" title=""><span class="name">SparkException</span></a><span class="result"> extends <span name="scala.Exception" class="extype">Exception</span> with <a href="SparkThrowable.html" name="org.apache.spark.SparkThrowable" id="org.apache.spark.SparkThrowable" class="extype">SparkThrowable</a></span></span></li><li class="indented0 " name="org.apache.spark.SparkExecutorInfo" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="SparkExecutorInfoextendsSerializable" class="anchorToMember"></a><a id="SparkExecutorInfo:SparkExecutorInfo" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkExecutorInfo.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="SparkExecutorInfo.html" title="Exposes information about Spark Executors."><span class="name">SparkExecutorInfo</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span><p class="shortcomment cmt">Exposes information about Spark Executors.</p><div class="fullcomment"><div class="comment cmt"><p>Exposes information about Spark Executors.</p><p>This interface is not designed to be implemented outside of Spark. We may add additional methods |
| which may break binary compatibility with outside implementations. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.SparkFirehoseListener" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="SparkFirehoseListenerextendsSparkListenerInterface" class="anchorToMember"></a><a id="SparkFirehoseListener:SparkFirehoseListener" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkFirehoseListener.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SparkFirehoseListener.html" title="Class that allows users to receive all SparkListener events."><span class="name">SparkFirehoseListener</span></a><span class="result"> extends <span name="org.apache.spark.scheduler.SparkListenerInterface" class="extype">SparkListenerInterface</span></span></span><p class="shortcomment cmt">Class that allows users to receive all SparkListener events.</p><div class="fullcomment"><div class="comment cmt"><p>Class that allows users to receive all SparkListener events. |
| Users should override the onEvent method.</p><p>This is a concrete Java class in order to ensure that we don't forget to update it when adding |
| new methods to SparkListener: forgetting to add a method will result in a compilation error (if |
| this was a concrete Scala class, default implementations of new event handlers would be inherited |
| from the SparkListener trait).</p><p>Please note until Spark 3.1.0 this was missing the DevelopApi annotation, this needs to be |
| taken into account if changing this API before a major release. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.SparkJobInfo" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="SparkJobInfoextendsSerializable" class="anchorToMember"></a><a id="SparkJobInfo:SparkJobInfo" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkJobInfo.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="SparkJobInfo.html" title="Exposes information about Spark Jobs."><span class="name">SparkJobInfo</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span><p class="shortcomment cmt">Exposes information about Spark Jobs.</p><div class="fullcomment"><div class="comment cmt"><p>Exposes information about Spark Jobs.</p><p>This interface is not designed to be implemented outside of Spark. We may add additional methods |
| which may break binary compatibility with outside implementations. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.SparkStageInfo" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="SparkStageInfoextendsSerializable" class="anchorToMember"></a><a id="SparkStageInfo:SparkStageInfo" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkStageInfo.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="SparkStageInfo.html" title="Exposes information about Spark Stages."><span class="name">SparkStageInfo</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span><p class="shortcomment cmt">Exposes information about Spark Stages.</p><div class="fullcomment"><div class="comment cmt"><p>Exposes information about Spark Stages.</p><p>This interface is not designed to be implemented outside of Spark. We may add additional methods |
| which may break binary compatibility with outside implementations. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.SparkStatusTracker" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="SparkStatusTrackerextendsAnyRef" class="anchorToMember"></a><a id="SparkStatusTracker:SparkStatusTracker" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkStatusTracker.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="SparkStatusTracker.html" title="Low-level status reporting APIs for monitoring job and stage progress."><span class="name">SparkStatusTracker</span></a><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">Low-level status reporting APIs for monitoring job and stage progress.</p><div class="fullcomment"><div class="comment cmt"><p>Low-level status reporting APIs for monitoring job and stage progress.</p><p>These APIs intentionally provide very weak consistency semantics; consumers of these APIs should |
| be prepared to handle empty / missing information. For example, a job's stage ids may be known |
| but the status API may not have any information about the details of those stages, so |
| <code>getStageInfo</code> could potentially return <code>None</code> for a valid stage id.</p><p>To limit memory usage, these APIs only provide information on recent jobs / stages. These APIs |
| will provide information for the last <code>spark.ui.retainedStages</code> stages and |
| <code>spark.ui.retainedJobs</code> jobs.</p><p>NOTE: this class's constructor should be considered private and may be subject to change. |
| </p></div></div></li><li class="indented0 " name="org.apache.spark.SparkThrowable" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="SparkThrowableextendsObject" class="anchorToMember"></a><a id="SparkThrowable:SparkThrowable" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkThrowable.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">trait</span></span> <span class="symbol"><a href="SparkThrowable.html" title="Interface mixed into Throwables thrown from Spark."><span class="name">SparkThrowable</span></a><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">Interface mixed into Throwables thrown from Spark.</p><div class="fullcomment"><div class="comment cmt"><p>Interface mixed into Throwables thrown from Spark.</p><p>- For backwards compatibility, existing Throwable types can be thrown with an arbitrary error |
| message with a null error class. See <a href="SparkException.html" name="org.apache.spark.SparkException" id="org.apache.spark.SparkException" class="extype">SparkException</a>. |
| - To promote standardization, Throwables should be thrown with an error class and message |
| parameters to construct an error message with SparkThrowableHelper.getMessage(). New Throwable |
| types should not accept arbitrary error messages. See <span name="SparkArithmeticException" class="extype">SparkArithmeticException</span>. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@Evolving</span><span class="args">()</span> </dd><dt>Since</dt><dd><p>3.2.0</p></dd></dl></div></li><li class="indented0 " name="org.apache.spark.TaskCommitDenied" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="TaskCommitDeniedextendsTaskFailedReasonwithProductwithSerializable" class="anchorToMember"></a><a id="TaskCommitDenied:TaskCommitDenied" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskCommitDenied.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case class</span></span> <span class="symbol"><a href="TaskCommitDenied.html" title=":: DeveloperApi :: Task requested the driver to commit, but was denied."><span class="name">TaskCommitDenied</span></a><span class="params">(<span name="jobID">jobID: <span name="scala.Int" class="extype">Int</span></span>, <span name="partitionID">partitionID: <span name="scala.Int" class="extype">Int</span></span>, <span name="attemptNumber">attemptNumber: <span name="scala.Int" class="extype">Int</span></span>)</span><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Task requested the driver to commit, but was denied.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Task requested the driver to commit, but was denied. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.TaskContext" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="TaskContextextendsSerializable" class="anchorToMember"></a><a id="TaskContext:TaskContext" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskContext.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">abstract </span> <span class="kind">class</span></span> <span class="symbol"><a href="TaskContext.html" title="Contextual information about a task which can be read or mutated during execution."><span class="name">TaskContext</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span><p class="shortcomment cmt">Contextual information about a task which can be read or mutated during |
| execution.</p><div class="fullcomment"><div class="comment cmt"><p>Contextual information about a task which can be read or mutated during |
| execution. To access the TaskContext for a running task, use:</p><pre>org.apache.spark.TaskContext.get()</pre></div></div></li><li class="indented0 " name="org.apache.spark.TaskEndReason" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="TaskEndReasonextendsAnyRef" class="anchorToMember"></a><a id="TaskEndReason:TaskEndReason" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskEndReason.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">sealed </span> <span class="kind">trait</span></span> <span class="symbol"><a href="TaskEndReason.html" title=":: DeveloperApi :: Various possible reasons why a task ended."><span class="name">TaskEndReason</span></a><span class="result"> extends <span name="scala.AnyRef" class="extype">AnyRef</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Various possible reasons why a task ended.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Various possible reasons why a task ended. The low-level TaskScheduler is supposed to retry |
| tasks several times for "ephemeral" failures, and only report back failures that require some |
| old stages to be resubmitted, such as shuffle map fetch failures. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.TaskFailedReason" group="Ungrouped" fullComment="yes" data-isabs="true" visbl="pub"><a id="TaskFailedReasonextendsTaskEndReason" class="anchorToMember"></a><a id="TaskFailedReason:TaskFailedReason" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskFailedReason.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier">sealed </span> <span class="kind">trait</span></span> <span class="symbol"><a href="TaskFailedReason.html" title=":: DeveloperApi :: Various possible reasons why a task failed."><span class="name">TaskFailedReason</span></a><span class="result"> extends <a href="TaskEndReason.html" name="org.apache.spark.TaskEndReason" id="org.apache.spark.TaskEndReason" class="extype">TaskEndReason</a></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Various possible reasons why a task failed.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Various possible reasons why a task failed. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.TaskKilled" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="TaskKilledextendsTaskFailedReasonwithProductwithSerializable" class="anchorToMember"></a><a id="TaskKilled:TaskKilled" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskKilled.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case class</span></span> <span class="symbol"><a href="TaskKilled.html" title=":: DeveloperApi :: Task was killed intentionally and needs to be rescheduled."><span class="name">TaskKilled</span></a><span class="params">(<span name="reason">reason: <span name="scala.Predef.String" class="extype">String</span></span>, <span name="accumUpdates">accumUpdates: <span name="scala.Seq" class="extype">Seq</span>[<a href="scheduler/AccumulableInfo.html" name="org.apache.spark.scheduler.AccumulableInfo" id="org.apache.spark.scheduler.AccumulableInfo" class="extype">AccumulableInfo</a>] = <span class="symbol">Seq.empty</span></span>, <span name="accums">accums: <span name="scala.Seq" class="extype">Seq</span>[<a href="util/AccumulatorV2.html" name="org.apache.spark.util.AccumulatorV2" id="org.apache.spark.util.AccumulatorV2" class="extype">AccumulatorV2</a>[_, _]] = <span class="symbol">Nil</span></span>, <span name="metricPeaks">metricPeaks: <span name="scala.Seq" class="extype">Seq</span>[<span name="scala.Long" class="extype">Long</span>] = <span class="symbol">Seq.empty</span></span>)</span><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Task was killed intentionally and needs to be rescheduled.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Task was killed intentionally and needs to be rescheduled. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.TaskKilledException" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="TaskKilledExceptionextendsRuntimeException" class="anchorToMember"></a><a id="TaskKilledException:TaskKilledException" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskKilledException.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="TaskKilledException.html" title=":: DeveloperApi :: Exception thrown when a task is explicitly killed (i.e., task failure is expected)."><span class="name">TaskKilledException</span></a><span class="result"> extends <span name="scala.RuntimeException" class="extype">RuntimeException</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Exception thrown when a task is explicitly killed (i.e., task failure is expected).</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Exception thrown when a task is explicitly killed (i.e., task failure is expected). |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li></ol></div><div id="deprecatedTypes" class="types members"><h3>Deprecated Type Members</h3><ol><li class="indented0 " name="org.apache.spark.ContextAwareIterator" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="ContextAwareIterator[+T]extendsIterator[T]" class="anchorToMember"></a><a id="ContextAwareIterator[+T]:ContextAwareIterator[T]" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/ContextAwareIterator.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">class</span></span> <span class="symbol"><a href="ContextAwareIterator.html" title=":: DeveloperApi :: A TaskContext aware iterator."><span class="name deprecated" title="Deprecated: (Since version 4.0.0) Only usage for Python evaluation is now extinct">ContextAwareIterator</span></a><span class="tparams">[<span name="T">+T</span>]</span><span class="result"> extends <span name="scala.Iterator" class="extype">Iterator</span>[<span name="org.apache.spark.ContextAwareIterator.T" class="extype">T</span>]</span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| A TaskContext aware iterator.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| A TaskContext aware iterator.</p><p>As the Python evaluation consumes the parent iterator in a separate thread, |
| it could consume more data from the parent even after the task ends and the parent is closed. |
| If an off-heap access exists in the parent iterator, it could cause segmentation fault |
| which crashes the executor. |
| Thus, we should use <a href="ContextAwareIterator.html" name="org.apache.spark.ContextAwareIterator" id="org.apache.spark.ContextAwareIterator" class="extype">ContextAwareIterator</a> to stop consuming after the task ends. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> <span class="name">@deprecated</span> </dd><dt>Deprecated</dt><dd class="cmt"><p><i>(Since version 4.0.0)</i> Only usage for Python evaluation is now extinct</p></dd><dt>Since</dt><dd><p>3.1.0</p></dd></dl></div></li></ol></div><div class="values members"><h3>Value Members</h3><ol><li class="indented0 " name="org.apache.spark#SPARK_BRANCH" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_BRANCH:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_BRANCH:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_BRANCH</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark#SPARK_BUILD_DATE" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_BUILD_DATE:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_BUILD_DATE:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_BUILD_DATE</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark#SPARK_BUILD_USER" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_BUILD_USER:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_BUILD_USER:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_BUILD_USER</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark#SPARK_DOC_ROOT" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_DOC_ROOT:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_DOC_ROOT:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_DOC_ROOT</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark#SPARK_REPO_URL" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_REPO_URL:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_REPO_URL:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_REPO_URL</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark#SPARK_REVISION" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_REVISION:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_REVISION:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_REVISION</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark#SPARK_VERSION" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_VERSION:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_VERSION:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_VERSION</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark#SPARK_VERSION_SHORT" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SPARK_VERSION_SHORT:String" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/index.html#SPARK_VERSION_SHORT:String" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">val</span></span> <span class="symbol"><span class="name">SPARK_VERSION_SHORT</span><span class="result">: <span name="scala.Predef.String" class="extype">String</span></span></span></li><li class="indented0 " name="org.apache.spark.BarrierTaskContext" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="BarrierTaskContext" class="anchorToMember"></a><a id="BarrierTaskContext:BarrierTaskContext" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/BarrierTaskContext$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="BarrierTaskContext$.html" title=""><span class="name">BarrierTaskContext</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span><div class="fullcomment"><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@Experimental</span><span class="args">()</span> <span class="name">@Since</span><span class="args">(<span><span class="symbol">"2.4.0"</span></span>)</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.Partitioner" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="Partitioner" class="anchorToMember"></a><a id="Partitioner:Partitioner" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/Partitioner$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="Partitioner$.html" title=""><span class="name">Partitioner</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span></li><li class="indented0 " name="org.apache.spark.Resubmitted" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="Resubmitted" class="anchorToMember"></a><a id="Resubmitted:Resubmitted" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/Resubmitted$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case object</span></span> <span class="symbol"><a href="Resubmitted$.html" title=":: DeveloperApi :: A org.apache.spark.scheduler.ShuffleMapTask that completed successfully earlier, but we lost the executor before the stage completed."><span class="name">Resubmitted</span></a><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| A <code>org.apache.spark.scheduler.ShuffleMapTask</code> that completed successfully earlier, but we |
| lost the executor before the stage completed.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| A <code>org.apache.spark.scheduler.ShuffleMapTask</code> that completed successfully earlier, but we |
| lost the executor before the stage completed. This means Spark needs to reschedule the task |
| to be re-executed on a different executor. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.SparkContext" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SparkContext" class="anchorToMember"></a><a id="SparkContext:SparkContext" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkContext$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="SparkContext$.html" title="The SparkContext object contains a number of implicit conversions and parameters for use with various Spark features."><span class="name">SparkContext</span></a><span class="result"> extends <span name="org.apache.spark.internal.Logging" class="extype">Logging</span></span></span><p class="shortcomment cmt">The SparkContext object contains a number of implicit conversions and parameters for use with |
| various Spark features.</p></li><li class="indented0 " name="org.apache.spark.SparkEnv" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SparkEnv" class="anchorToMember"></a><a id="SparkEnv:SparkEnv" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkEnv$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="SparkEnv$.html" title=""><span class="name">SparkEnv</span></a><span class="result"> extends <span name="org.apache.spark.internal.Logging" class="extype">Logging</span></span></span></li><li class="indented0 " name="org.apache.spark.SparkException" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SparkException" class="anchorToMember"></a><a id="SparkException:SparkException" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkException$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="SparkException$.html" title=""><span class="name">SparkException</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span></li><li class="indented0 " name="org.apache.spark.SparkFiles" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="SparkFiles" class="anchorToMember"></a><a id="SparkFiles:SparkFiles" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/SparkFiles$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="SparkFiles$.html" title="Resolves paths to files added through SparkContext.addFile()."><span class="name">SparkFiles</span></a></span><p class="shortcomment cmt">Resolves paths to files added through <code>SparkContext.addFile()</code>.</p></li><li class="indented0 " name="org.apache.spark.Success" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="Success" class="anchorToMember"></a><a id="Success:Success" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/Success$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case object</span></span> <span class="symbol"><a href="Success$.html" title=":: DeveloperApi :: Task succeeded."><span class="name">Success</span></a><span class="result"> extends <a href="TaskEndReason.html" name="org.apache.spark.TaskEndReason" id="org.apache.spark.TaskEndReason" class="extype">TaskEndReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| Task succeeded.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| Task succeeded. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.TaskContext" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="TaskContext" class="anchorToMember"></a><a id="TaskContext:TaskContext" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskContext$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="TaskContext$.html" title=""><span class="name">TaskContext</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span></li><li class="indented0 " name="org.apache.spark.TaskResultLost" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="TaskResultLost" class="anchorToMember"></a><a id="TaskResultLost:TaskResultLost" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/TaskResultLost$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case object</span></span> <span class="symbol"><a href="TaskResultLost$.html" title=":: DeveloperApi :: The task finished successfully, but the result was lost from the executor's block manager before it was fetched."><span class="name">TaskResultLost</span></a><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| The task finished successfully, but the result was lost from the executor's block manager before |
| it was fetched.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| The task finished successfully, but the result was lost from the executor's block manager before |
| it was fetched. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.UnknownReason" group="Ungrouped" fullComment="yes" data-isabs="false" visbl="pub"><a id="UnknownReason" class="anchorToMember"></a><a id="UnknownReason:UnknownReason" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/UnknownReason$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">case object</span></span> <span class="symbol"><a href="UnknownReason$.html" title=":: DeveloperApi :: We don't know why the task ended -- for example, because of a ClassNotFound exception when deserializing the task result."><span class="name">UnknownReason</span></a><span class="result"> extends <a href="TaskFailedReason.html" name="org.apache.spark.TaskFailedReason" id="org.apache.spark.TaskFailedReason" class="extype">TaskFailedReason</a> with <span name="scala.Product" class="extype">Product</span> with <span name="scala.Serializable" class="extype">Serializable</span></span></span><p class="shortcomment cmt">:: DeveloperApi :: |
| We don't know why the task ended -- for example, because of a ClassNotFound exception when |
| deserializing the task result.</p><div class="fullcomment"><div class="comment cmt"><p>:: DeveloperApi :: |
| We don't know why the task ended -- for example, because of a ClassNotFound exception when |
| deserializing the task result. |
| </p></div><dl class="attributes block"><dt>Annotations</dt><dd><span class="name">@DeveloperApi</span><span class="args">()</span> </dd></dl></div></li><li class="indented0 " name="org.apache.spark.WritableConverter" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="WritableConverter" class="anchorToMember"></a><a id="WritableConverter:WritableConverter" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/WritableConverter$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="WritableConverter$.html" title=""><span class="name">WritableConverter</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span></li><li class="indented0 " name="org.apache.spark.WritableFactory" group="Ungrouped" fullComment="no" data-isabs="false" visbl="pub"><a id="WritableFactory" class="anchorToMember"></a><a id="WritableFactory:WritableFactory" class="anchorToMember"></a> <span class="permalink"><a href="../../../org/apache/spark/WritableFactory$.html" title="Permalink"><i class="material-icons"></i></a></span> <span class="modifier_kind"><span class="modifier"></span> <span class="kind">object</span></span> <span class="symbol"><a href="WritableFactory$.html" title=""><span class="name">WritableFactory</span></a><span class="result"> extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/Serializable.html#java.io.Serializable" name="java.io.Serializable" id="java.io.Serializable" class="extype">Serializable</a></span></span></li></ol></div></div><div id="inheritedMembers"><div name="scala.AnyRef" class="parent"><h3>Inherited from <span name="scala.AnyRef" class="extype">AnyRef</span></h3></div><div name="scala.Any" class="parent"><h3>Inherited from <span name="scala.Any" class="extype">Any</span></h3></div></div><div id="groupedMembers"><div name="Ungrouped" class="group"><h3>Ungrouped</h3></div></div></div><div id="tooltip"></div><div id="footer"></div></body></div></div></div></body></html> |
