Google Git
Sign in
apache / spark-website / asf-site / . / site / releases / spark-release-3-1-1.html
blob: 437dc793abf68bc7518644eda6787a5c2ffe3c5c [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>
Spark Release 3.1.1 | Apache Spark
</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
<link href="/css/custom.css" rel="stylesheet">
<!-- Code highlighter CSS -->
<link href="/css/pygments-default.css" rel="stylesheet">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '40']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="global">
<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
<a class="navbar-brand" href="/">
<img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">
<ul class="navbar-nav me-auto">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Libraries
</a>
<ul class="dropdown-menu" aria-labelledby="libraries">
<li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
<li><a class="dropdown-item" href="/spark-connect/">Spark Connect</a></li>
<li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
<li><a class="dropdown-item" href="/pandas-on-spark/">pandas on Spark</a></li>
<li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
<li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
<li>
<hr class="dropdown-divider">
</li>
<li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Documentation
</a>
<ul class="dropdown-menu" aria-labelledby="documentation">
<li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
<li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
<li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Community
</a>
<ul class="dropdown-menu" aria-labelledby="community">
<li><a class="dropdown-item" href="/community.html">Mailing Lists &amp; Resources</a></li>
<li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
<li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
</li>
<li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
</li>
<li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
<li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
<li><a class="dropdown-item" href="/history.html">Project History</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Developers
</a>
<ul class="dropdown-menu" aria-labelledby="developers">
<li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
<li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
<li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
<li><a class="dropdown-item" href="/security.html">Security</a></li>
</ul>
</li>
</ul>
<ul class="navbar-nav ml-auto">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
data-bs-toggle="dropdown" aria-expanded="false">
Apache Software Foundation
</a>
<ul class="dropdown-menu" aria-labelledby="apacheFoundation">
<li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
<li><a class="dropdown-item"
href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/security/">Security</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
</ul>
</li>
</ul>
</div>
</nav>
<div class="container">
<div class="row mt-4">
<div class="col-12 col-md-9">
<h2>Spark Release 3.1.1</h2>
<p>Apache Spark 3.1.1 is the second release of the 3.x line. This release adds Python type annotations and Python dependency management support as part of Project Zen. Other major updates include improved ANSI SQL compliance support, history server support in structured streaming, the general availability (GA) of Kubernetes and node decommissioning in Kubernetes and Standalone. In addition, this release continues to focus on usability, stability, and polish while resolving around 1500 tickets.</p>
<p>To download Apache Spark 3.1.1, visit the <a href="https://spark.apache.org/downloads.html">downloads</a> page. You can consult JIRA for the <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315420&amp;version=12349541">detailed changes</a>. We have curated a list of high level changes here, grouped by major modules.</p>
<ul id="markdown-toc">
<li><a href="#core-and-spark-sql" id="markdown-toc-core-and-spark-sql">Core and Spark SQL</a></li>
<li><a href="#pyspark" id="markdown-toc-pyspark">PySpark</a></li>
<li><a href="#structured-streaming" id="markdown-toc-structured-streaming">Structured Streaming</a></li>
<li><a href="#mllib" id="markdown-toc-mllib">MLlib</a></li>
<li><a href="#sparkr" id="markdown-toc-sparkr">SparkR</a></li>
<li><a href="#graphx" id="markdown-toc-graphx">GraphX</a></li>
<li><a href="#deprecations-and-removals" id="markdown-toc-deprecations-and-removals">Deprecations and Removals</a></li>
<li><a href="#known-issues" id="markdown-toc-known-issues">Known Issues</a></li>
<li><a href="#credits" id="markdown-toc-credits">Credits</a></li>
</ul>
<h3 id="core-and-spark-sql">Core and Spark SQL</h3>
<p><strong>Highlight</strong></p>
<ul>
<li>Unify create table SQL syntax (<a href="https://issues.apache.org/jira/browse/SPARK-31257">SPARK-31257</a>)</li>
<li>Shuffled hash join improvement (<a href="https://issues.apache.org/jira/browse/SPARK-32461">SPARK-32461</a>)</li>
<li>Experimental node decommissioning for Kubernates and Standalone (<a href="https://issues.apache.org/jira/browse/SPARK-20624">SPARK-20624</a>)</li>
<li>Enhanced subexpression elimination (<a href="https://issues.apache.org/jira/browse/SPARK-33092">SPARK-33092</a>, <a href="https://issues.apache.org/jira/browse/SPARK-33337">SPARK-33337</a>, <a href="https://issues.apache.org/jira/browse/SPARK-33427">SPARK-33427</a>, <a href="https://issues.apache.org/jira/browse/SPARK-33540">SPARK-33540</a>)</li>
<li>Kubernetes GA (<a href="https://issues.apache.org/jira/browse/SPARK-33005">SPARK-33005</a>)</li>
<li>Use Apache Hadoop 3.2.0 by default (<a href="https://issues.apache.org/jira/browse/SPARK-32058">SPARK-32058</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32841">SPARK-32841</a>)</li>
</ul>
<p><strong>ANSI SQL Compatibility Enhancements</strong></p>
<ul>
<li>Support char/varchar data type (<a href="https://issues.apache.org/jira/browse/SPARK-33480">SPARK-33480</a>)</li>
<li>ANSI mode: runtime errors instead of returning null (<a href="https://issues.apache.org/jira/browse/SPARK-33275">SPARK-33275</a>)</li>
<li>ANSI mode: new explicit cast syntax rules (<a href="https://issues.apache.org/jira/browse/SPARK-33354">SPARK-33354</a>)</li>
<li>Add SQL standard command SET TIME ZONE (<a href="https://issues.apache.org/jira/browse/SPARK-32272">SPARK-32272</a>)</li>
<li>Unify create table SQL syntax (<a href="https://issues.apache.org/jira/browse/SPARK-31257">SPARK-31257</a>)</li>
<li>Unify temp view and permanent view behaviors (<a href="https://issues.apache.org/jira/browse/SPARK-33138">SPARK-33138</a>)</li>
<li>Support column list in INSERT statement (<a href="https://issues.apache.org/jira/browse/SPARK-32976">SPARK-32976</a>)</li>
<li>Support ANSI nested bracketed comments (<a href="https://issues.apache.org/jira/browse/SPARK-28880">SPARK-28880</a>)</li>
</ul>
<p><strong>Performance Enhancements</strong></p>
<ul>
<li>Host-local shuffle data reading without shuffle service (<a href="https://issues.apache.org/jira/browse/SPARK-32077">SPARK-32077</a>)</li>
<li>Remove redundant sorts before repartition nodes (<a href="https://issues.apache.org/jira/browse/SPARK-32276">SPARK-32276</a>)</li>
<li>Partially push down predicates (<a href="https://issues.apache.org/jira/browse/SPARK-32302">SPARK-32302</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32352">SPARK-32352</a>)</li>
<li>Push down filters through expand (<a href="https://issues.apache.org/jira/browse/SPARK-33302">SPARK-33302</a>)</li>
<li>Push more possible predicates through Join via CNF conversion (<a href="https://issues.apache.org/jira/browse/SPARK-31705">SPARK-31705</a>)</li>
<li>Remove shuffle by preserving output partitioning of broadcast hash join (<a href="https://issues.apache.org/jira/browse/SPARK-31869">SPARK-31869</a>)</li>
<li>Remove shuffle by improving reordering join keys (<a href="https://issues.apache.org/jira/browse/SPARK-32282">SPARK-32282</a>)</li>
<li>Remove shuffle by normalizing output partitioning and sortorder (<a href="https://issues.apache.org/jira/browse/SPARK-33399">SPARK-33399</a>)</li>
<li>Shuffled hash join improvement (<a href="https://issues.apache.org/jira/browse/SPARK-32461">SPARK-32461</a>)
<ul>
<li>Preserve shuffled hash join build side partitioning (<a href="https://issues.apache.org/jira/browse/SPARK-32330">SPARK-32330</a>)</li>
<li>Preserve hash join (BHJ and SHJ) stream side ordering (<a href="https://issues.apache.org/jira/browse/SPARK-32383">SPARK-32383</a>)</li>
<li>Coalesce bucketed tables for shuffled hash join (<a href="https://issues.apache.org/jira/browse/SPARK-32286">SPARK-32286</a>)</li>
<li>Add code-gen for shuffled hash join (<a href="https://issues.apache.org/jira/browse/SPARK-32421">SPARK-32421</a>)</li>
<li>Support full outer join in shuffled hash join (<a href="https://issues.apache.org/jira/browse/SPARK-32399">SPARK-32399</a>)</li>
</ul>
</li>
<li>Support subexpression elimination in project with whole-stage-codegen (<a href="https://issues.apache.org/jira/browse/SPARK-33092">SPARK-33092</a>)</li>
<li>Support subexpression elimination in conditional expressions (<a href="https://issues.apache.org/jira/browse/SPARK-33337">SPARK-33337</a>)</li>
<li>Support subexpression elimination for interpreted expression evaluation (<a href="https://issues.apache.org/jira/browse/SPARK-33427">SPARK-33427</a>)</li>
<li>Support subexpression elimination for interpreted predicate (<a href="https://issues.apache.org/jira/browse/SPARK-33540">SPARK-33540</a>)</li>
<li>Other optimizer rules
<ul>
<li>Rule ExtractSingleColumnNullAwareAntiJoin (<a href="https://issues.apache.org/jira/browse/SPARK-32290">SPARK-32290</a>)</li>
<li>Rule EliminateNullAwareAntiJoin (<a href="https://issues.apache.org/jira/browse/SPARK-32573">SPARK-32573</a>)</li>
<li>Rule EliminateAggregateFilter (<a href="https://issues.apache.org/jira/browse/SPARK-32540">SPARK-32540</a>)</li>
<li>Rule UnwrapCastInBinaryComparison (<a href="https://issues.apache.org/jira/browse/SPARK-32858">SPARK-32858</a>)</li>
<li>Rule DisableUnnecessaryBucketedScan (<a href="https://issues.apache.org/jira/browse/SPARK-32859">SPARK-32859</a>)</li>
<li>Rule CoalesceBucketsInJoin (<a href="https://issues.apache.org/jira/browse/SPARK-31350">SPARK-31350</a>)</li>
<li>Prune unnecessary nested fields from generate without project (<a href="https://issues.apache.org/jira/browse/SPARK-29721">SPARK-29721</a>)</li>
<li>Prune unnecessary nested fields from aggregate and expand (<a href="https://issues.apache.org/jira/browse/SPARK-27217">SPARK-27217</a>)</li>
<li>Prune unnecessary nested fields from repartition-by-expression and join (<a href="https://issues.apache.org/jira/browse/SPARK-31736">SPARK-31736</a>)</li>
<li>Prune unnecessary nested fields over cosmetic variations (<a href="https://issues.apache.org/jira/browse/SPARK-32163">SPARK-32163</a>)</li>
<li>Prune unnecessary nested fields from window and sort (<a href="https://issues.apache.org/jira/browse/SPARK-32059">SPARK-32059</a>)</li>
<li>Optimize size of CreateArray/CreateMap to be the size of its children (<a href="https://issues.apache.org/jira/browse/SPARK-33544">SPARK-33544</a>)</li>
</ul>
</li>
</ul>
<p><strong>Extensibility Enhancements</strong></p>
<ul>
<li>Stage Level Resource configuration and Scheduling (<a href="https://issues.apache.org/jira/browse/SPARK-27495">SPARK-27495</a>)
<ul>
<li>Expose RDD APIs for Stage Level Scheduling (<a href="https://issues.apache.org/jira/browse/SPARK-29150">SPARK-29150</a>)</li>
<li>Merge resource profiles within a stage (<a href="https://issues.apache.org/jira/browse/SPARK-29153">SPARK-29153</a>)</li>
<li>Stage level scheduling support for Kubernetes (<a href="https://issues.apache.org/jira/browse/SPARK-33288">SPARK-33288</a>)</li>
<li>Add UI support for stage level scheduling (<a href="https://issues.apache.org/jira/browse/SPARK-29303">SPARK-29303</a>)</li>
</ul>
</li>
<li>Remote storage for persisting shuffle data (<a href="https://issues.apache.org/jira/browse/SPARK-25299">SPARK-25299</a>)
<ul>
<li>Shuffle writer metadata APIs (<a href="https://issues.apache.org/jira/browse/SPARK-31798">SPARK-31798</a>)</li>
</ul>
</li>
<li>Allow to use a custom shuffle manager with external shuffle service (<a href="https://issues.apache.org/jira/browse/SPARK-33037">SPARK-33037</a>)</li>
<li>Add SupportsPartitions APIs on DataSourceV2 (<a href="https://issues.apache.org/jira/browse/SPARK-31694">SPARK-31694</a>)</li>
<li>Add SupportsMetadataColumns API on DataSourceV2 (<a href="https://issues.apache.org/jira/browse/SPARK-31255">SPARK-31255</a>)</li>
<li>Make SQL cache serialization pluggable (<a href="https://issues.apache.org/jira/browse/SPARK-32274">SPARK-32274</a>)</li>
<li>Introduce the &#8220;purge&#8221; option in TableCatalog.dropTable for v2 catalog (<a href="https://issues.apache.org/jira/browse/SPARK-33364">SPARK-33364</a>)</li>
</ul>
<p><strong>Connector Enhancements</strong></p>
<ul>
<li>Hive Metastore partition filter pushdown improvement (<a href="https://issues.apache.org/jira/browse/SPARK-33537">SPARK-33537</a>)
<ul>
<li>Support contains, starts-with and ends-with filters (<a href="https://issues.apache.org/jira/browse/SPARK-33458">SPARK-33458</a>)</li>
<li>Support filter by date type (<a href="https://issues.apache.org/jira/browse/SPARK-33477">SPARK-33477</a>)</li>
<li>Support filter by not-equals (<a href="https://issues.apache.org/jira/browse/SPARK-33582">SPARK-33582</a>)</li>
</ul>
</li>
<li>Parquet
<ul>
<li>Allow complex type in map’s key type in Parquet (<a href="https://issues.apache.org/jira/browse/SPARK-32639">SPARK-32639</a>)</li>
<li>Allow saving/loading INT96 in Parquet without rebasing (<a href="https://issues.apache.org/jira/browse/SPARK-33160">SPARK-33160</a>)</li>
</ul>
</li>
<li>ORC
<ul>
<li>Nested column predicate pushdown for ORC (<a href="https://issues.apache.org/jira/browse/SPARK-25557">SPARK-25557</a>)</li>
<li>Upgrade Apache ORC to 1.5.12 (<a href="https://issues.apache.org/jira/browse/SPARK-33050">SPARK-33050</a>)</li>
</ul>
</li>
<li>CSV
<ul>
<li>Leverage SQL text data source during CSV schema inference (<a href="https://issues.apache.org/jira/browse/SPARK-32270">SPARK-32270</a>)</li>
</ul>
</li>
<li>JSON
<ul>
<li>Support filters pushdown in JSON datasource (<a href="https://issues.apache.org/jira/browse/SPARK-30648">SPARK-30648</a>)</li>
</ul>
</li>
<li>JDBC
<ul>
<li>Implement catalog APIs for JDBC (<a href="https://issues.apache.org/jira/browse/SPARK-32375">SPARK-32375</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32579">SPARK-32579</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32402">SPARK-32402</a>, <a href="https://issues.apache.org/jira/browse/SPARK-33130">SPARK-33130</a>)</li>
<li>Create JDBC authentication provider developer API (<a href="https://issues.apache.org/jira/browse/SPARK-32001">SPARK-32001</a>)</li>
<li>Add JDBC connection provider disable possibility (<a href="https://issues.apache.org/jira/browse/SPARK-32047">SPARK-32047</a>)</li>
</ul>
</li>
<li>Avro
<ul>
<li>Support filters pushdown in Avro datasource (<a href="https://issues.apache.org/jira/browse/SPARK-32346">SPARK-32346</a>)</li>
</ul>
</li>
</ul>
<p><strong>Feature Enhancements</strong></p>
<ul>
<li>Node Decommissioning (<a href="https://issues.apache.org/jira/browse/SPARK-20624">SPARK-20624</a>)
<ul>
<li>Basic framework (<a href="https://issues.apache.org/jira/browse/SPARK-20628">SPARK-20628</a>)</li>
<li>Migrate RDD blocks during decommission(<a href="https://issues.apache.org/jira/browse/SPARK-20732">SPARK-20732</a>)</li>
<li>Graceful decommissioning as part of dynamic scaling (<a href="https://issues.apache.org/jira/browse/SPARK-31198">SPARK-31198</a>)</li>
<li>Migrate shuffle blocks during decommission (<a href="https://issues.apache.org/jira/browse/SPARK-20629">SPARK-20629</a>)</li>
<li>Only exit executor when tasks and block migration are finished (<a href="https://issues.apache.org/jira/browse/SPARK-31197">SPARK-31197</a>)</li>
<li>Support fallback storage during decommission (<a href="https://issues.apache.org/jira/browse/SPARK-33545">SPARK-33545</a>)</li>
</ul>
</li>
<li>New built-in functions
<ul>
<li>json_array_length (<a href="https://issues.apache.org/jira/browse/SPARK-31008">SPARK-31008</a>)</li>
<li>json_object_keys (<a href="https://issues.apache.org/jira/browse/SPARK-31009">SPARK-31009</a>)</li>
<li>current_catalog (<a href="https://issues.apache.org/jira/browse/SPARK-30352">SPARK-30352</a>)</li>
<li>timestamp_seconds, timestamp_millis, timestamp_micros (<a href="https://issues.apache.org/jira/browse/SPARK-31710">SPARK-31710</a>)</li>
<li>width_bucket (<a href="https://issues.apache.org/jira/browse/SPARK-21117">SPARK-21117</a>)</li>
<li>regexp_extract_all (<a href="https://issues.apache.org/jira/browse/SPARK-24884">SPARK-24884</a>)</li>
<li>nth_value (<a href="https://issues.apache.org/jira/browse/SPARK-27951">SPARK-27951</a>)</li>
<li>raise_error (<a href="https://issues.apache.org/jira/browse/SPARK-32793">SPARK-32793</a>)</li>
<li>unix_seconds, unix_millis and unix_micros (<a href="https://issues.apache.org/jira/browse/SPARK-33627">SPARK-33627</a>)</li>
<li>date_from_unix_date and unix_date (<a href="https://issues.apache.org/jira/browse/SPARK-33646">SPARK-33646</a>)</li>
<li>current_timezone (<a href="https://issues.apache.org/jira/browse/SPARK-33469">SPARK-33469</a>)</li>
</ul>
</li>
<li>EXPLAIN command enhancement (<a href="https://issues.apache.org/jira/browse/SPARK-32337">SPARK-32337</a>, <a href="https://issues.apache.org/jira/browse/SPARK-31325">SPARK-31325</a>)</li>
<li>Provide a option to disable user supplied Hints (<a href="https://issues.apache.org/jira/browse/SPARK-31875">SPARK-31875</a>)</li>
<li>Support Hive style REPLACE COLUMNS syntax (<a href="https://issues.apache.org/jira/browse/SPARK-30613">SPARK-30613</a>)</li>
<li>Support &#8216;LIKE ANY&#8217; and &#8216;LIKE ALL&#8217; operators (<a href="https://issues.apache.org/jira/browse/SPARK-30724">SPARK-30724</a>)</li>
<li>Support unlimited MATCHED and NOT MATCHED in MERGE INTO (<a href="https://issues.apache.org/jira/browse/SPARK-32030">SPARK-32030</a>)</li>
<li>Support &#8216;F&#8217;-suffixed float literals (<a href="https://issues.apache.org/jira/browse/SPARK-32207">SPARK-32207</a>)</li>
<li>Support RESET syntax to reset single configuration (<a href="https://issues.apache.org/jira/browse/SPARK-32406">SPARK-32406</a>)</li>
<li>Support filter expression allows simultaneous use of DISTINCT (<a href="https://issues.apache.org/jira/browse/SPARK-30276">SPARK-30276</a>)</li>
<li>Support alter table add/drop partition command for DSv2 (<a href="https://issues.apache.org/jira/browse/SPARK-32512">SPARK-32512</a>)</li>
<li>Support NOT IN subqueries inside nested OR conditions (<a href="https://issues.apache.org/jira/browse/SPARK-25154">SPARK-25154</a>)</li>
<li>Support REFRESH FUNCTION command (<a href="https://issues.apache.org/jira/browse/SPARK-31999">SPARK-31999</a>)</li>
<li>Add &#8216;sameSemantics&#8217; and &#8216;sementicHash&#8217; methods in Dataset (<a href="https://issues.apache.org/jira/browse/SPARK-30791">SPARK-30791</a>)</li>
<li>Support composed type of case class in UDF (<a href="https://issues.apache.org/jira/browse/SPARK-31826">SPARK-31826</a>)</li>
<li>Support enumeration in encoders (<a href="https://issues.apache.org/jira/browse/SPARK-32585">SPARK-32585</a>)</li>
<li>Support nested field APIs withField and dropFields (<a href="https://issues.apache.org/jira/browse/SPARK-31317">SPARK-31317</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32511">SPARK-32511</a>)</li>
<li>Support to fill nulls for missing columns in unionByName (<a href="https://issues.apache.org/jira/browse/SPARK-29358">SPARK-29358</a>)</li>
<li>Support DataFrameReader.table to take the specified options (<a href="https://issues.apache.org/jira/browse/SPARK-32592">SPARK-32592</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32844">SPARK-32844</a>)</li>
<li>Support HDFS location in <code class="language-plaintext highlighter-rouge">spark.sql.hive.metastore.jars</code> (<a href="https://issues.apache.org/jira/browse/SPARK-32852">SPARK-32852</a>)</li>
<li>Support &#8211;archives option natively (<a href="https://issues.apache.org/jira/browse/SPARK-33530">SPARK-33530</a>, <a href="https://issues.apache.org/jira/browse/SPARK-33615">SPARK-33615</a>)</li>
<li>Enhance ExecutorPlugin API to include methods for task start and end events (<a href="https://issues.apache.org/jira/browse/SPARK-33088">SPARK-33088</a>)</li>
</ul>
<p><strong>Other Notable Changes</strong></p>
<ul>
<li>Provide Search Function in Spark docs site (<a href="https://issues.apache.org/jira/browse/SPARK-33166">SPARK-33166</a>)</li>
<li>Use Apache Hadoop 3.2.0 by default (<a href="https://issues.apache.org/jira/browse/SPARK-32058">SPARK-32058</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32841">SPARK-32841</a>)</li>
<li>Upgrade Apache Arrow to 2.0.0 (<a href="https://issues.apache.org/jira/browse/SPARK-33213">SPARK-33213</a>)</li>
<li>Kubernetes GA (<a href="https://issues.apache.org/jira/browse/SPARK-33005">SPARK-33005</a>)
<ul>
<li>Adds support for Kubernetes NFS volume mounts (<a href="https://issues.apache.org/jira/browse/SPARK-31394">SPARK-31394</a>)</li>
<li>Support dynamic PVC creation/deletion (<a href="https://issues.apache.org/jira/browse/SPARK-32971">SPARK-32971</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32997">SPARK-32997</a>)</li>
<li>Respect environment variables and configurations for Python executables (<a href="https://issues.apache.org/jira/browse/SPARK-33748">SPARK-33748</a>)</li>
<li>Support Python dependency (<a href="https://issues.apache.org/jira/browse/SPARK-33748">SPARK-27936</a>)</li>
<li>Make pod allocation executor timeouts configurable and allow scheduling with pending pods (<a href="https://issues.apache.org/jira/browse/SPARK-33231">SPARK-33231</a>, <a href="https://issues.apache.org/jira/browse/SPARK-33262">SPARK-33262</a>)</li>
<li>Respect executor idle timeout conf in ExecutorPodsAllocator (<a href="https://issues.apache.org/jira/browse/SPARK-33099">SPARK-33099</a>)</li>
<li>Support JDBC Kerberos with keytab (<a href="https://issues.apache.org/jira/browse/SPARK-12312">SPARK-12312</a>)</li>
</ul>
</li>
<li>Enable Java 8 time API in thrift server (<a href="https://issues.apache.org/jira/browse/SPARK-31910">SPARK-31910</a>)</li>
<li>Enable Java 8 time API in UDFs (<a href="https://issues.apache.org/jira/browse/SPARK-32154">SPARK-32154</a>)</li>
<li>Overflow check for aggregate sum with decimals (<a href="https://issues.apache.org/jira/browse/SPARK-28067">SPARK-28067</a>)</li>
<li>Fix commit collision in dynamic partition overwrite mode (<a href="https://issues.apache.org/jira/browse/SPARK-27194">SPARK-27194</a>, <a href="https://issues.apache.org/jira/browse/SPARK-29302">SPARK-29302</a>)</li>
<li>Removed references to slave, blacklist and whitelist (<a href="https://issues.apache.org/jira/browse/SPARK-32004">SPARK-32004</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32036">SPARK-32036</a>, <a href="https://issues.apache.org/jira/browse/SPARK-32037">SPARK-32037</a>)</li>
<li>Remove task result size check for shuffle map stage (<a href="https://issues.apache.org/jira/browse/SPARK-32470">SPARK-32470</a>)</li>
<li>Generalize ExecutorSource to expose user-given file system schemes (<a href="https://issues.apache.org/jira/browse/SPARK-33476">SPARK-33476</a>)</li>
<li>Add StorageLevel.DISK_ONLY_3 (<a href="https://issues.apache.org/jira/browse/SPARK-32517">SPARK-32517</a>)</li>
<li>Expose executor memory metrics in the web UI for executors (<a href="https://issues.apache.org/jira/browse/SPARK-23432">SPARK-23432</a>)</li>
<li>Expose executor memory metrics at the stage level, in the Stages tab (<a href="https://issues.apache.org/jira/browse/SPARK-26341">SPARK-26341</a>)</li>
<li>Fix explicitly set of <code class="language-plaintext highlighter-rouge">spark.ui.port</code> in YARN cluster mode (<a href="https://issues.apache.org/jira/browse/SPARK-29465">SPARK-29465</a>)</li>
<li>Add <code class="language-plaintext highlighter-rouge">spark.submit.waitForCompletion</code> configuration to control spark-submit exit in Standalone cluster mode (<a href="https://issues.apache.org/jira/browse/SPARK-31486">SPARK-31486</a>)</li>
<li>Do not propagate Hadoop’s classpath for Spark distribution with built-in Hadoop (<a href="https://issues.apache.org/jira/browse/SPARK-31960">SPARK-31960</a>)</li>
<li>Fix jobs disappear intermittently in SHS under high load (<a href="https://issues.apache.org/jira/browse/SPARK-33841">SPARK-33841</a>)</li>
<li>Redact sensitive attributes of application log in SHS (<a href="https://issues.apache.org/jira/browse/SPARK-33504">SPARK-33504</a>)</li>
<li>Set up yarn.Client to print direct links to driver stdout/stderr (<a href="https://issues.apache.org/jira/browse/SPARK-33185">SPARK-33185</a>)</li>
<li>Fix memory leak when fail to store pieces of broadcast (<a href="https://issues.apache.org/jira/browse/SPARK-32715">SPARK-32715</a>)</li>
<li>Make BlockManagerMaster driver heartbeat timeout configurable (<a href="https://issues.apache.org/jira/browse/SPARK-34278">SPARK-34278</a>)</li>
<li>Unify and complete cache behaviors (<a href="https://issues.apache.org/jira/browse/SPARK-33507">SPARK-33507</a>)</li>
</ul>
<p><strong>Changes of behavior</strong></p>
<p>Please read the migration guides for each component: <a href="https://spark.apache.org/docs/3.1.1/core-migration-guide.html">Spark Core</a> and <a href="https://spark.apache.org/docs/3.1.1/sql-migration-guide.html">Spark SQL</a>.</p>
<p><em>Programming guides: <a href="https://spark.apache.org/docs/3.1.1/rdd-programming-guide.html">Spark RDD Programming Guide</a> and <a href="https://spark.apache.org/docs/3.1.1/sql-programming-guide.html">Spark SQL, DataFrames and Datasets Guide</a>.</em></p>
<h3 id="pyspark">PySpark</h3>
<p><strong>Project Zen</strong></p>
<ul>
<li>Project Zen: Improving Python usability (<a href="https://issues.apache.org/jira/browse/SPARK-32082">SPARK-32082</a>)</li>
<li>PySpark type hints support (<a href="https://issues.apache.org/jira/browse/SPARK-32681">SPARK-32681</a>)</li>
<li>Redesign PySpark documentation (<a href="https://issues.apache.org/jira/browse/SPARK-31851">SPARK-31851</a>)</li>
<li>Migrate to NumPy documentation style (<a href="https://issues.apache.org/jira/browse/SPARK-32085">SPARK-32085</a>)</li>
<li>Installation option for PyPI Users (<a href="https://issues.apache.org/jira/browse/SPARK-32017">SPARK-32017</a>)</li>
<li>Un-deprecate inferring DataFrame schema from list of dict (<a href="https://issues.apache.org/jira/browse/SPARK-32686">SPARK-32686</a>)</li>
<li>Simplify the exception message from Python UDFs (<a href="https://issues.apache.org/jira/browse/SPARK-33407">SPARK-33407</a>)</li>
</ul>
<p><strong>Other Notable Changes</strong></p>
<ul>
<li>Stage Level Scheduling APIs (<a href="https://issues.apache.org/jira/browse/SPARK-29641">SPARK-29641</a>)</li>
<li>Deduplicate deterministic PythonUDF calls (<a href="https://issues.apache.org/jira/browse/SPARK-33303">SPARK-33303</a>)</li>
<li>Support higher order functions in PySpark functions(<a href="https://issues.apache.org/jira/browse/SPARK-30681">SPARK-30681</a>)</li>
<li>Support data source v2x write APIs (<a href="https://issues.apache.org/jira/browse/SPARK-29157">SPARK-29157</a>)</li>
<li>Support percentile_approx in PySpark functions(<a href="https://issues.apache.org/jira/browse/SPARK-30569">SPARK-30569</a>)</li>
<li>Support inputFiles in PySpark DataFrame (<a href="https://issues.apache.org/jira/browse/SPARK-31763">SPARK-31763</a>)</li>
<li>Support withField in PySpark Column (<a href="https://issues.apache.org/jira/browse/SPARK-32835">SPARK-32835</a>)</li>
<li>Support dropFields in PySpark Column (<a href="https://issues.apache.org/jira/browse/SPARK-32511">SPARK-32511</a>)</li>
<li>Support nth_value in PySpark functions (<a href="https://issues.apache.org/jira/browse/SPARK-33020">SPARK-33020</a>)</li>
<li>Support acosh, asinh and atanh (<a href="https://issues.apache.org/jira/browse/SPARK-33563">SPARK-33563</a>)</li>
<li>Support getCheckpointDir method in PySpark SparkContext (<a href="https://issues.apache.org/jira/browse/SPARK-33017">SPARK-33017</a>)</li>
<li>Support to fill nulls for missing columns in unionByName (<a href="https://issues.apache.org/jira/browse/SPARK-32798">SPARK-32798</a>)</li>
<li>Update cloudpickle to v1.5.0 (<a href="https://issues.apache.org/jira/browse/SPARK-32094">SPARK-32094</a>)</li>
<li>Add MapType support for PySpark with Arrow (<a href="https://issues.apache.org/jira/browse/SPARK-24554">SPARK-24554</a>)</li>
<li>DataStreamReader.table and DataStreamWriter.toTable (<a href="https://issues.apache.org/jira/browse/SPARK-33836">SPARK-33836</a>)</li>
</ul>
<p><strong>Changes of behavior</strong></p>
<p>Please read the migration guides for <a href="https://spark.apache.org/docs/3.1.1/api/python/migration_guide/index.html">PySpark</a>.</p>
<p><em>Programming guides: <a href="https://spark.apache.org/docs/3.1.1/api/python/getting_started/index.html">PySpark Getting Started</a> and <a href="https://spark.apache.org/docs/3.1.1/api/python/user_guide/index.html">PySpark User Guide</a>.</em></p>
<h3 id="structured-streaming">Structured Streaming</h3>
<p><strong>Performance Enhancements</strong></p>
<ul>
<li>Cache fetched list of files beyond maxFilesPerTrigger as unread file (<a href="https://issues.apache.org/jira/browse/SPARK-30866">SPARK-30866</a>)</li>
<li>Streamline the logic on file stream source and sink metadata log (<a href="https://issues.apache.org/jira/browse/SPARK-30462">SPARK-30462</a>)</li>
<li>Avoid reading compact metadata log twice if the query restarts from compact batch (<a href="https://issues.apache.org/jira/browse/SPARK-30900">SPARK-30900</a>)</li>
</ul>
<p><strong>Feature Enhancements</strong></p>
<ul>
<li>Add DataStreamReader.table API (<a href="https://issues.apache.org/jira/browse/SPARK-32885">SPARK-32885</a>)</li>
<li>Add DataStreamWriter.toTable API (<a href="https://issues.apache.org/jira/browse/SPARK-32896">SPARK-32896</a>)</li>
<li>Left semi stream-stream join (<a href="https://issues.apache.org/jira/browse/SPARK-32862">SPARK-32862</a>)</li>
<li>Full outer stream-stream join (<a href="https://issues.apache.org/jira/browse/SPARK-32863">SPARK-32863</a>)</li>
<li>Provide a new option to have retention on output files (<a href="https://issues.apache.org/jira/browse/SPARK-27188">SPARK-27188</a>)</li>
<li>Add Spark Structured Streaming History Server Support (<a href="https://issues.apache.org/jira/browse/SPARK-31953">SPARK-31953</a>)</li>
<li>Introduce State schema validation among query restart (<a href="https://issues.apache.org/jira/browse/SPARK-27237">SPARK-27237</a>)</li>
</ul>
<p><strong>Other Notable Changes</strong></p>
<ul>
<li>Introduce schema validation for streaming state store (<a href="https://issues.apache.org/jira/browse/SPARK-31894">SPARK-31894</a>)</li>
<li>Support to use a different compression codec in state store (<a href="https://issues.apache.org/jira/browse/SPARK-33263">SPARK-33263</a>)</li>
<li>Kafka connector infinite wait because metadata never updated (<a href="https://issues.apache.org/jira/browse/SPARK-28367">SPARK-28367</a>)</li>
<li>Upgrade Kafka to 2.6.0 (<a href="https://issues.apache.org/jira/browse/SPARK-32568">SPARK-32568</a>)</li>
<li>Pagination support for Structured Streaming UI pages (<a href="https://issues.apache.org/jira/browse/SPARK-31642">SPARK-31642</a>, <a href="https://issues.apache.org/jira/browse/SPARK-30119">SPARK-30119</a>)</li>
<li>State information in Structured Streaming UI (<a href="https://issues.apache.org/jira/browse/SPARK-33223">SPARK-33223</a>)</li>
<li>Watermark gap information in Structured Streaming UI (<a href="https://issues.apache.org/jira/browse/SPARK-33224">SPARK-33224</a>)</li>
<li>Expose state custom metrics information on SS UI (<a href="https://issues.apache.org/jira/browse/SPARK-33287">SPARK-33287</a>)</li>
<li>Add a new metric regarding number of rows later than watermark (<a href="https://issues.apache.org/jira/browse/SPARK-24634">SPARK-24634</a>)</li>
</ul>
<p><strong>Changes of behavior</strong></p>
<p>Please read the migration guides for <a href="https://spark.apache.org/docs/3.1.1/ss-migration-guide.html">Structured Streaming</a>.</p>
<p><em>Programming guides: <a href="https://spark.apache.org/docs/3.1.1/structured-streaming-programming-guide.html">Structured Streaming Programming Guide</a>.</em></p>
<h3 id="mllib">MLlib</h3>
<p><strong>Highlight</strong></p>
<ul>
<li>LinearSVC blockify input vectors (<a href="https://issues.apache.org/jira/browse/SPARK-30642">SPARK-30642</a>)</li>
<li>LogisticRegression blockify input vectors (<a href="https://issues.apache.org/jira/browse/SPARK-30659">SPARK-30659</a>)</li>
<li>LinearRegression blockify input vectors (<a href="https://issues.apache.org/jira/browse/SPARK-30660">SPARK-30660</a>)</li>
<li>AFT blockify input vectors (<a href="https://issues.apache.org/jira/browse/SPARK-31656">SPARK-31656</a>)</li>
<li>Add support for association rules in ML (<a href="https://issues.apache.org/jira/browse/SPARK-19939">SPARK-19939</a>)</li>
<li>Add training summary for LinearSVCModel (<a href="https://issues.apache.org/jira/browse/SPARK-20249">SPARK-20249</a>)</li>
<li>Add summary to RandomForestClassificationModel (<a href="https://issues.apache.org/jira/browse/SPARK-23631">SPARK-23631</a>)</li>
<li>Add training summary to FMClassificationModel (<a href="https://issues.apache.org/jira/browse/SPARK-32140">SPARK-32140</a>)</li>
<li>Add summary to MultilayerPerceptronClassificationModel (<a href="https://issues.apache.org/jira/browse/SPARK-32449">SPARK-32449</a>)</li>
<li>Add FMClassifier to SparkR (<a href="https://issues.apache.org/jira/browse/SPARK-30820">SPARK-30820</a>)</li>
<li>Add SparkR LinearRegression wrapper (<a href="https://issues.apache.org/jira/browse/SPARK-30818">SPARK-30818</a>)</li>
<li>Add FMRegressor wrapper to SparkR (<a href="https://issues.apache.org/jira/browse/SPARK-30819">SPARK-30819</a>)</li>
<li>Add SparkR wrapper for vector_to_array (<a href="https://issues.apache.org/jira/browse/SPARK-33040">SPARK-33040</a>)</li>
<li>adaptively blockify instances - LinearSVC (<a href="https://issues.apache.org/jira/browse/SPARK-32907">SPARK-32907</a>)</li>
<li>make CrossValidator/TrainValidateSplit/OneVsRest Reader/Writer support Python backend estimator/evaluator (<a href="https://issues.apache.org/jira/browse/SPARK-33520">SPARK-33520</a>)</li>
<li>Improve performance of ML ALS recommendForAll by GEMV (<a href="https://issues.apache.org/jira/browse/SPARK-33520">SPARK-33518</a>)</li>
<li>Add UnivariateFeatureSelector (<a href="https://issues.apache.org/jira/browse/SPARK-34080">SPARK-34080</a>)</li>
</ul>
<p><strong>Other Notable Changes</strong></p>
<ul>
<li>GMM compute summary and update distributions in one job (<a href="https://issues.apache.org/jira/browse/SPARK-31032">SPARK-31032</a>)</li>
<li>Remove ChiSqSelector dependency on mllib.ChiSqSelectorModel (<a href="https://issues.apache.org/jira/browse/SPARK-31077">SPARK-31077</a>)</li>
<li>Flatten the result dataframe of tests in testChiSquare (<a href="https://issues.apache.org/jira/browse/SPARK-31301">SPARK-31301</a>)</li>
<li>MinHash keyDistance optimization (<a href="https://issues.apache.org/jira/browse/SPARK-31436">SPARK-31436</a>)</li>
<li>KMeans optimization based on triangle-inequality (<a href="https://issues.apache.org/jira/browse/SPARK-31007">SPARK-31007</a>)</li>
<li>Add weight support in ClusteringEvaluator (<a href="https://issues.apache.org/jira/browse/SPARK-31734">SPARK-31734</a>)</li>
<li>Add getMetrics in Evaluators (<a href="https://issues.apache.org/jira/browse/SPARK-31768">SPARK-31768</a>)</li>
<li>Add instance weight support in LinearRegressionSummary (<a href="https://issues.apache.org/jira/browse/SPARK-31944">SPARK-31944</a>)</li>
<li>Add user-specified fold column to CrossValidator (<a href="https://issues.apache.org/jira/browse/SPARK-31777">SPARK-31777</a>)</li>
<li>ML params default value parity in feature and tuning (<a href="https://issues.apache.org/jira/browse/SPARK-32310">SPARK-32310</a>)</li>
<li>Fix double caching in KMeans/BiKMeans (<a href="https://issues.apache.org/jira/browse/SPARK-32676">SPARK-32676</a>)</li>
<li>aft transform optimization (<a href="https://issues.apache.org/jira/browse/SPARK-33111">SPARK-33111</a>)</li>
<li>FeatureHasher transform optimization (<a href="https://issues.apache.org/jira/browse/SPARK-32974">SPARK-32974</a>)</li>
<li>Add array_to_vector function for dataframe column (<a href="https://issues.apache.org/jira/browse/SPARK-33556">SPARK-33556</a>)</li>
<li>ML params default value parity in classification, regression, clustering and fpm (<a href="https://issues.apache.org/jira/browse/SPARK-32310">SPARK-32310</a>)</li>
<li>Summary.totalIterations greater than maxIters (<a href="https://issues.apache.org/jira/browse/SPARK-31925">SPARK-31925</a>)</li>
<li>tree models prediction optimization (<a href="https://issues.apache.org/jira/browse/SPARK-32298">SPARK-32298</a>)</li>
</ul>
<p><strong>Changes of behavior</strong></p>
<p>Please read the migration guides for <a href="https://spark.apache.org/docs/3.1.1/ml-migration-guide.html">MLlib</a>.</p>
<p><em>Programming guide: <a href="https://spark.apache.org/docs/3.1.1/ml-guide.html">Machine Learning Library (MLlib) Guide</a>.</em></p>
<h3 id="sparkr">SparkR</h3>
<ul>
<li>Add SparkR interface for higher order functions (<a href="https://issues.apache.org/jira/browse/SPARK-30682">SPARK-30682</a>)</li>
<li>Support to fill nulls for missing columns in unionByName (<a href="https://issues.apache.org/jira/browse/SPARK-32798">SPARK-32798</a>)</li>
<li>Support withColumn in SparkR functions (<a href="https://issues.apache.org/jira/browse/SPARK-32946">SPARK-32946</a>)</li>
<li>Support timestamp_seconds in SparkR functions (<a href="https://issues.apache.org/jira/browse/SPARK-32949">SPARK-32949</a>)</li>
<li>Support nth_value in SparkR functions (<a href="https://issues.apache.org/jira/browse/SPARK-33030">SPARK-33030</a>)</li>
<li>Minimum Arrow version bumped up to 1.0.0 (<a href="https://issues.apache.org/jira/browse/SPARK-32452">SPARK-32452</a>)</li>
<li>Support array_to_vector in SparkR functions (<a href="https://issues.apache.org/jira/browse/SPARK-33622">SPARK-33622</a>)</li>
<li>Support acosh, asinh and atanh (<a href="https://issues.apache.org/jira/browse/SPARK-33563">SPARK-33563</a>)</li>
<li>Support from_avro and to_avro (<a href="https://issues.apache.org/jira/browse/SPARK-33304">SPARK-33304</a>)</li>
</ul>
<p><strong>Changes of behavior</strong></p>
<p>Please read the <a href="https://spark.apache.org/docs/3.1.1/sparkr-migration-guide.html">migration guide</a> for details.</p>
<p><em>Programming guide: <a href="https://spark.apache.org/docs/3.1.1/sparkr.html">SparkR (R on Spark)</a>.</em></p>
<h3 id="graphx">GraphX</h3>
<p><em>Programming guide: <a href="https://spark.apache.org/docs/3.1.1/graphx-programming-guide.html">GraphX Programming Guide</a>.</em></p>
<h3 id="deprecations-and-removals">Deprecations and Removals</h3>
<ul>
<li>Drop Python 2.7, 3.4 and 3.5 (<a href="https://issues.apache.org/jira/browse/SPARK-32138">SPARK-32138</a>)</li>
<li>Drop R &lt; 3.5 support (<a href="https://issues.apache.org/jira/browse/SPARK-32073">SPARK-32073</a>)</li>
<li>Remove hive-1.2 distribution (<a href="https://issues.apache.org/jira/browse/SPARK-32981">SPARK-32981</a>)</li>
<li>Remove references to org.spark-project.hive (<a href="https://issues.apache.org/jira/browse/SPARK-20202">SPARK-20202</a>)</li>
<li>Deprecate support of multiple workers on the same host in Standalone (<a href="https://issues.apache.org/jira/browse/SPARK-31018">SPARK-31018</a>)</li>
</ul>
<h2 id="known-issues">Known Issues</h2>
<ul>
<li>[<a href="https://issues.apache.org/jira/browse/SPARK-34543">SPARK-33392</a>] Respect case sensitivity in V1 ALTER TABLE .. SET LOCATION</li>
<li>[<a href="https://issues.apache.org/jira/browse/SPARK-34531">SPARK-34531</a>] Remove Experimental API tag in PrometheusServlet</li>
<li>[<a href="https://issues.apache.org/jira/browse/SPARK-34515">SPARK-34515</a>] Fix NPE if InSet contains null value during getPartitionsByFilter</li>
<li>[<a href="https://issues.apache.org/jira/browse/SPARK-34497">SPARK-34497</a>] JDBC connection provider is not removing kerberos credentials from JVM security context</li>
<li>[<a href="https://issues.apache.org/jira/browse/SPARK-34490">SPARK-34490</a>] table maybe resolved as a view if the table is dropped</li>
<li>[<a href="https://issues.apache.org/jira/browse/SPARK-34473">SPARK-34473</a>] avoid NPE in DataFrameReader.schema(StructType)</li>
<li>[<a href="https://issues.apache.org/jira/browse/SPARK-34436">SPARK-34436</a>] DPP support LIKE ANY/ALL</li>
</ul>
<h2 id="credits">Credits</h2>
<p>Last but not least, this release would not have been possible without the following contributors: Abhishek Dixit, Adam Binford, Ajith S, Akshat Bordia, Alessandro Patti, Alex Favaro, Ali Afroozeh, Ali Smesseim, Allison Wang, Ankit Srivastava, Anton Okolnychyi, Antonin Delpeuch, Artsiom Yudovin, Arwin Tio, Attila Zsolt Piros, Baohe Zhang, Bo Yang, Bo Zhang, Brandon Jiang, Bruce Robbins, Bryan Cutler, CC Highman, Chandni Singh, Chao Sun, Chen Zhang, Cheng Su, Chuliang Xiao, DB Tsai, Dale Clarke, Daniel Himmelstein, Daniel Moore, David Toneian, Denis Pyshev, Devesh Agrawal, Dilip Biswal, Dmitry Sabanin, Dongjoon Hyun, Du Ripeng, Emilian Bold, Eren Avsarogullari, Eric Lemmon, Eric Wu, Erik Krogen, Fabian Höring, Farhan Khan, Farooq Qaiser, Fei Wang, Fokko Driesprong, Frank Yin, Fuwang Hu, Gabor Somogyi, Gengliang Wang, Gera Shegalov, German Schiavon Matteo, Goki Mori, Guangxin Wang, Gustavo Martin Morcuende, Herman Van Hovell, Holden Karau, Huang Yi, Haejoon Lee, Huaxin Gao, Hyukjin Kwon, Izek Greenfield, Jackey Lee, Jacob Kim, Jalpan Randeri, Jatin Puri, Jiaan Geng, Jinxin Tang, Josh Soref, Jonathan Lafleche, Jungtaek Lim, Karen Feng, Karol Chmist, Kaxil Naik, Kazuaki Ishizaki, Ke Jia, Keiji Yoshida, Kent Yao, Kevin Su, Kevin Wang, Koert Kuipers, Kousuke Saruta, Kyle Bendickson, Lantao Jin, Leanken Lin, Liang Zhang, Liang-Chi Hsieh, Linhong Liu, Lipeng Zhu, Lu Lu, Luca Canali, Maciej Szymkiewicz, Manu Zhang, Marcelo Vanzin, Maryann Xue, Matthew Cheah, Maxim Gekk, Michael Chirico, Michael Munday, Michał Wieleba, Min Shen, Nan Zhu, Nicholas Chammas, Nicholas Marcott, Nik Vanderhoof, Onur Satici, Pablo Langa, Pascal Gillet, Paul Reidy, Pavithra Ramachandran, Pedro Rossi, Peter Toth, Philipse Guo, Piotr Grzegorski, Prakhar Jain, Prashant Sharma, Qianyang Yu, Qilong Su, Rajat Ahuja, Rakesh Raushan, Rameshkrishnan Muthusamy, Raphael Auv, Richard Penney, Robert (Bobby) Evans, Rohit Mishra,Ruifeng Zheng, Ryan Blue, Ryotaro Tsuzuki, Samir Khan, Samuel Souza, Sandeep Katta, Sander Goos, Saurabh Chawla, Sean Owen, Seongjin Cho, Shane Knapp, Shanyu Zhao, Shaoyun Chen, Shixiong Zhu, Shruti Gumma, Srinivas Rishindra Pothireddi, Stavros Kontopoulos, StefanXiepj, Stijn De Haes, Stuart White, Sudharshann D, Sunitha Kambhampati, Takeshi Yamamuro, Takuya UESHIN, Tanel Kiis, Tathagata Das, Terry Kim, Thomas Graves, Tianshi Zhu, Tom Howland, Tom Van Bussel, Udbhav Agrawal, Uncle Gen, Utkarsh Agarwal, Venkata Krishnan Sowrirajan, Vlad Glinsky, Warren Zhu, Weichen Xu, Wenchen Fan, William Hyun, Wing Yew Poon, Xianyin Xin, Xiao Li, Xiduo You, Xingbo Jiang, Xinrong Meng, Xinyi Yu, Xuedong Luan, Yang Jie, Yaroslav Tkachenko, Ye Zhou, Yi Wu, Yi Zhu, Yu Zhong, Yuanjian Li, Yuexin Zhang, Yuming Wang, Yuning Zhang, Zhen Li, Zhenhua Wang, Zhicheng Jin, Zirui Xu, Zuo Dao, akiyamaneko, artiship, cristichircu, dzlab, huangtianhua, liucht-inspur, manubatham20, waitinfuture, wang-zhun, yzjg</p>
<p>
<br/>
<a href="/news/">Spark News Archive</a>
</p>
</div>
<div class="col-12 col-md-3">
<div class="news" style="margin-bottom: 20px;">
<h5>Latest News</h5>
<ul class="list-unstyled">
<li><a href="/news/spark-3-4-3-released.html">Spark 3.4.3 released</a>
<span class="small">(Apr 18, 2024)</span></li>
<li><a href="/news/spark-3-5-1-released.html">Spark 3.5.1 released</a>
<span class="small">(Feb 23, 2024)</span></li>
<li><a href="/news/spark-3-3-4-released.html">Spark 3.3.4 released</a>
<span class="small">(Dec 16, 2023)</span></li>
<li><a href="/news/spark-3-4-2-released.html">Spark 3.4.2 released</a>
<span class="small">(Nov 30, 2023)</span></li>
</ul>
<p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
</div>
<div style="text-align:center; margin-bottom: 20px;">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
</a>
</div>
<div class="hidden-xs hidden-sm">
<a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
Download Spark
</a>
<p style="font-size: 16px; font-weight: 500; color: #555;">
Built-in Libraries:
</p>
<ul class="list-none">
<li><a href="/sql/">SQL and DataFrames</a></li>
<li><a href="/streaming/">Spark Streaming</a></li>
<li><a href="/mllib/">MLlib (machine learning)</a></li>
<li><a href="/graphx/">GraphX (graph)</a></li>
</ul>
<a href="/third-party-projects.html">Third-Party Projects</a>
</div>
</div>
</div>
<footer class="small">
<hr>
Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
All other marks mentioned may be trademarks or registered trademarks of their respective owners.
Copyright &copy; 2018 The Apache Software Foundation, Licensed under the
<a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
</footer>
</div>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js"
integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM"
crossorigin="anonymous"></script>
<script src="https://code.jquery.com/jquery.js"></script>
<script src="/js/lang-tabs.js"></script>
<script src="/js/downloads.js"></script>
</body>
</html>