Google Git
Sign in
apache / spark-website / asf-site / . / site / releases / spark-release-3-5-1.html
blob: 03e99b828a31d2ea7235bacc786ce50c7bf04c0b [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>
Spark Release 3.5.1 | Apache Spark
</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
<link href="/css/custom.css" rel="stylesheet">
<!-- Code highlighter CSS -->
<link href="/css/pygments-default.css" rel="stylesheet">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '40']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="global">
<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
<a class="navbar-brand" href="/">
<img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">
<ul class="navbar-nav me-auto">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Libraries
</a>
<ul class="dropdown-menu" aria-labelledby="libraries">
<li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
<li><a class="dropdown-item" href="/spark-connect/">Spark Connect</a></li>
<li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
<li><a class="dropdown-item" href="/pandas-on-spark/">pandas on Spark</a></li>
<li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
<li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
<li>
<hr class="dropdown-divider">
</li>
<li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Documentation
</a>
<ul class="dropdown-menu" aria-labelledby="documentation">
<li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
<li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
<li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Community
</a>
<ul class="dropdown-menu" aria-labelledby="community">
<li><a class="dropdown-item" href="/community.html">Mailing Lists &amp; Resources</a></li>
<li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
<li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
</li>
<li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
</li>
<li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
<li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
<li><a class="dropdown-item" href="/history.html">Project History</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Developers
</a>
<ul class="dropdown-menu" aria-labelledby="developers">
<li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
<li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
<li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
<li><a class="dropdown-item" href="/security.html">Security</a></li>
</ul>
</li>
</ul>
<ul class="navbar-nav ml-auto">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
data-bs-toggle="dropdown" aria-expanded="false">
Apache Software Foundation
</a>
<ul class="dropdown-menu" aria-labelledby="apacheFoundation">
<li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
<li><a class="dropdown-item"
href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/security/">Security</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
</ul>
</li>
</ul>
</div>
</nav>
<div class="container">
<div class="row mt-4">
<div class="col-12 col-md-9">
<h2>Spark Release 3.5.1</h2>
<p>Spark 3.5.1 is the first maintenance release containing security and correctness fixes. This release is based on the branch-3.5 maintenance branch of Spark. We strongly recommend all 3.5 users to upgrade to this stable release.</p>
<h3 id="notable-changes">Notable changes</h3>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45187">[SPARK-45187]</a>: Fix WorkerPage to use the same pattern for <code class="language-plaintext highlighter-rouge">logPage</code> urls</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45553">[SPARK-45553]</a>: Deprecate assertPandasOnSparkEqual</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45652">[SPARK-45652]</a>: SPJ: Handle empty input partitions after dynamic filtering</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46012">[SPARK-46012]</a>: EventLogFileReader should not read rolling logs if appStatus is missing</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46029">[SPARK-46029]</a>: Escape the single quote, _ and % for DS V2 pushdown</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46369">[SPARK-46369]</a>: Remove <code class="language-plaintext highlighter-rouge">kill</code> link from RELAUNCHING drivers in MasterPage</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46704">[SPARK-46704]</a>: Fix <code class="language-plaintext highlighter-rouge">MasterPage</code> to sort <code class="language-plaintext highlighter-rouge">Running Drivers</code> table by <code class="language-plaintext highlighter-rouge">Duration</code> column correctly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46817">[SPARK-46817]</a>: Fix <code class="language-plaintext highlighter-rouge">spark-daemon.sh</code> usage by adding <code class="language-plaintext highlighter-rouge">decommission</code> command</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46888">[SPARK-46888]</a>: Fix <code class="language-plaintext highlighter-rouge">Master</code> to reject worker kill request if decommission is disabled</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-39910">[SPARK-39910]</a>: DataFrameReader API cannot read files from hadoop archives (.har)</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-40154">[SPARK-40154]</a>: PySpark: DataFrame.cache docstring gives wrong storage level</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43393">[SPARK-43393]</a>: Sequence expression can overflow</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44683">[SPARK-44683]</a>: Logging level isn&#8217;t passed to RocksDB state store provider correctly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44805">[SPARK-44805]</a>: Data lost after union using spark.sql.parquet.enableNestedColumnVectorizedReader=true</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44840">[SPARK-44840]</a>: array_insert() give wrong results for ngative index</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44910">[SPARK-44910]</a>: Encoders.bean does not support superclasses with generic type arguments</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44973">[SPARK-44973]</a>: Fix ArrayIndexOutOfBoundsException in conv()</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45014">[SPARK-45014]</a>: Clean up fileserver when cleaning up files, jars and archives in SparkContext</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45057">[SPARK-45057]</a>: Deadlock caused by rdd replication level of 2</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45072">[SPARK-45072]</a>: Fix Outerscopes for same cell evaluation</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45075">[SPARK-45075]</a>: Alter table with invalid default value will not report error</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45078">[SPARK-45078]</a>: The ArrayInsert function should make explicit casting when element type not equals derived component type</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45081">[SPARK-45081]</a>: Encoders.bean does no longer work with read-only properties</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45106">[SPARK-45106]</a>: percentile_cont gets internal error when user input fails runtime replacement&#8217;s input type check</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45117">[SPARK-45117]</a>: Implement missing otherCopyArgs for the MultiCommutativeOp expression</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45124">[SPARK-45124]</a>: Do not use local user ID for Local Relations</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45132">[SPARK-45132]</a>: Fix IDENTIFIER clause for functions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45171">[SPARK-45171]</a>: GenerateExec fails to initialize non-deterministic expressions before use</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45182">[SPARK-45182]</a>: Ignore task completion from old stage after retrying indeterminate stages</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45205">[SPARK-45205]</a>: Since version 3.2.0, Spark SQL has taken longer to execute &#8220;show paritions&#8221;,probably because of changes introduced by SPARK-35278</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45227">[SPARK-45227]</a>: Fix a subtle thread-safety issue with CoarseGrainedExecutorBackend where an executor process randomly gets stuck</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45291">[SPARK-45291]</a>: Use unknown query execution id instead of no such app when id is invalid</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45311">[SPARK-45311]</a>: Encoder fails on many &#8220;NoSuchElementException: None.get&#8221; since 3.4.x, search for an encoder for a generic type, and since 3.5.x isn&#8217;t &#8220;an expression encoder&#8221;</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45346">[SPARK-45346]</a>: Parquet schema inference should respect case sensitive flag when merging schema</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45371">[SPARK-45371]</a>: FIx shading problem in Spark Connect</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45383">[SPARK-45383]</a>: Missing case for RelationTimeTravel in CheckAnalysis</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45389">[SPARK-45389]</a>: Correct MetaException matching rule on getting partition metadata</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45424">[SPARK-45424]</a>: Regression in CSV schema inference when timestamps do not match specified timestampFormat</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45430">[SPARK-45430]</a>: FramelessOffsetWindowFunctionFrame fails when ignore nulls and offset &gt; # of rows</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45433">[SPARK-45433]</a>: CSV/JSON schema inference when timestamps do not match specified timestampFormat with only one row on each partition report error</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45449">[SPARK-45449]</a>: Cache Invalidation Issue with JDBC Table</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45484">[SPARK-45484]</a>: Fix the bug that uses incorrect parquet compression codec lz4raw</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45498">[SPARK-45498]</a>: Followup: Ignore task completion from old stage after retrying indeterminate stages</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45508">[SPARK-45508]</a>: Add &#8220;&#8211;add-opens=java.base/jdk.internal.ref=ALL-UNNAMED&#8221; so Platform can access cleaner on Java 9+</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45543">[SPARK-45543]</a>: InferWindowGroupLimit causes bug if the other window functions haven&#8217;t the same window frame as the rank-like functions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45561">[SPARK-45561]</a>: Convert TINYINT catalyst properly in MySQL Dialect</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45580">[SPARK-45580]</a>: Subquery changes the output schema of the outer query</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45584">[SPARK-45584]</a>: Execution fails when there are subqueries in TakeOrderedAndProjectExec</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45592">[SPARK-45592]</a>: AQE and InMemoryTableScanExec correctness bug</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45604">[SPARK-45604]</a>: Converting timestamp_ntz to array<timestamp_ntz> can cause NPE or SEGFAULT on parquet vectorized reader</timestamp_ntz></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45616">[SPARK-45616]</a>: Usages of ParVector are unsafe because it does not propagate ThreadLocals or SparkSession</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45631">[SPARK-45631]</a>: Broken backward compatibility in PySpark: StreamingQueryListener due to the addition of onQueryIdle</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45670">[SPARK-45670]</a>: SparkSubmit does not support &#8211;total-executor-cores when deploying on K8s</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45678">[SPARK-45678]</a>: Cover BufferReleasingInputStream.available under tryOrFetchFailedException</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45786">[SPARK-45786]</a>: Inaccurate Decimal multiplication and division results</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45814">[SPARK-45814]</a>: ArrowConverters.createEmptyArrowBatch may cause memory leak</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45896">[SPARK-45896]</a>: Expression encoding fails for Seq/Map of <code class="language-plaintext highlighter-rouge">Option[Seq/Date/Timestamp/BigDecimal]</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45920">[SPARK-45920]</a>: group by ordinal should be idempotent</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46006">[SPARK-46006]</a>: YarnAllocator miss clean targetNumExecutorsPerResourceProfileId after YarnSchedulerBackend call stop</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46016">[SPARK-46016]</a>: Fix pandas API support list properly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46062">[SPARK-46062]</a>: CTE reference node does not inherit the flag <code class="language-plaintext highlighter-rouge">isStreaming</code> from CTE definition node</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46064">[SPARK-46064]</a>: EliminateEventTimeWatermark does not consider the fact that isStreaming flag can change for current child during resolution</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46092">[SPARK-46092]</a>: Overflow in Parquet row group filter creation causes incorrect results</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46189">[SPARK-46189]</a>: Various Pandas functions fail in interpreted mode</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46239">[SPARK-46239]</a>: Hide Jetty info</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46274">[SPARK-46274]</a>: Range operator computeStats() proper long conversions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46275">[SPARK-46275]</a>: Protobuf: Permissive mode should return null rather than struct with null fields</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46330">[SPARK-46330]</a>: Loading of Spark UI blocks for a long time when HybridStore enabled</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46339">[SPARK-46339]</a>: Directory with number name should not be treated as metadata log</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46388">[SPARK-46388]</a>: HiveAnalysis misses pattern guard <code class="language-plaintext highlighter-rouge">query.resolved</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46396">[SPARK-46396]</a>: LegacyFastTimestampFormatter.parseOptional should not throw exception</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46443">[SPARK-46443]</a>: Decimal precision and scale should decided by JDBC dialect.</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46453">[SPARK-46453]</a>: SessionHolder doesn&#8217;t throw exceptions from internalError()</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46466">[SPARK-46466]</a>: vectorized parquet reader should never do rebase for timestamp ntz</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46480">[SPARK-46480]</a>: Fix NPE when table cache task attempt</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46535">[SPARK-46535]</a>: NPE when describe extended a column without col stats</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46562">[SPARK-46562]</a>: Remove retrieval of <code class="language-plaintext highlighter-rouge">keytabFile</code> from <code class="language-plaintext highlighter-rouge">UserGroupInformation</code> in <code class="language-plaintext highlighter-rouge">HiveAuthFactory</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46590">[SPARK-46590]</a>: Coalesce partiton assert error after skew join optimization</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46598">[SPARK-46598]</a>: OrcColumnarBatchReader should respect the memory mode when creating column vectors for the missing column</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46602">[SPARK-46602]</a>: CREATE VIEW IF NOT EXISTS should never throw <code class="language-plaintext highlighter-rouge">TABLE_OR_VIEW_ALREADY_EXISTS</code> exception</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46609">[SPARK-46609]</a>: avoid exponential explosion in PartitioningPreservingUnaryExecNode</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46640">[SPARK-46640]</a>: RemoveRedundantAliases does not account for SubqueryExpression when removing aliases</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46663">[SPARK-46663]</a>: Disable memory profiler for pandas UDFs with iterators</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46676">[SPARK-46676]</a>: dropDuplicatesWithinWatermark throws error on canonicalizing plan</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46684">[SPARK-46684]</a>: CoGroup.applyInPandas/Arrow should pass arguments properly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46700">[SPARK-46700]</a>: count the last spilling for the shuffle disk spilling bytes metric</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46747">[SPARK-46747]</a>: Too Many Shared Locks due to PostgresDialect.getTableExistsQuery - LIMIT 1</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46763">[SPARK-46763]</a>: ReplaceDeduplicateWithAggregate fails when non-grouping keys have duplicate attributes</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46769">[SPARK-46769]</a>: Refine timestamp related schema inference</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46779">[SPARK-46779]</a>: Grouping by subquery with a cached relation can fail</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46786">[SPARK-46786]</a>: Fix MountVolumesFeatureStep to use ReadWriteOncePod instead of ReadWriteOnce</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46794">[SPARK-46794]</a>: Incorrect results due to inferred predicate from checkpoint with subquery</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46796">[SPARK-46796]</a>: RocksDB versionID Mismatch in SST files</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46855">[SPARK-46855]</a>: Add <code class="language-plaintext highlighter-rouge">sketch</code> to the dependencies of the <code class="language-plaintext highlighter-rouge">catalyst</code> module in <code class="language-plaintext highlighter-rouge">module.py</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46861">[SPARK-46861]</a>: Avoid Deadlock in DAGScheduler</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46862">[SPARK-46862]</a>: Incorrect count() of a dataframe loaded from CSV datasource</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46945">[SPARK-46945]</a>: Add <code class="language-plaintext highlighter-rouge">spark.kubernetes.legacy.useReadWriteOnceAccessMode</code> for old K8s clusters</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47019">[SPARK-47019]</a>: AQE dynamic cache partitioning causes SortMergeJoin to result in data loss</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45360">[SPARK-45360]</a>: Initialize spark session builder configuration from SPARK_REMOTE</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45706">[SPARK-45706]</a>: Makes entire Binder build fails fast during setting up</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46732">[SPARK-46732]</a>: Propagate JobArtifactSet to broadcast execution thread</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44833">[SPARK-44833]</a>: Spark Connect reattach when initial ExecutePlan didn&#8217;t reach server doing too eager Reattach</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44835">[SPARK-44835]</a>: SparkConnect ReattachExecute could raise before ExecutePlan even attaches.</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45050">[SPARK-45050]</a>: Improve error message for UNKNOWN io.grpc.StatusRuntimeException</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45071">[SPARK-45071]</a>: Optimize the processing speed of <code class="language-plaintext highlighter-rouge">BinaryArithmetic#dataType</code> when processing multi-column data</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45250">[SPARK-45250]</a>: Support stage level task resource profile for yarn cluster when dynamic allocation disabled</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45386">[SPARK-45386]</a>: Correctness issue when persisting using StorageLevel.NONE</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45419">[SPARK-45419]</a>: Avoid reusing rocksdb sst files in a dfferent rocksdb instance by removing file version map entry of larger versions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45495">[SPARK-45495]</a>: Support stage level task resource profile for k8s cluster when dynamic allocation disabled</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45538">[SPARK-45538]</a>: pyspark connect overwrite_partitions bug</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45770">[SPARK-45770]</a>: Fix column resolution in DataFrame.drop</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45882">[SPARK-45882]</a>: BroadcastHashJoinExec propagate partitioning should respect CoalescedHashPartitioning</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45974">[SPARK-45974]</a>: Add scan.filterAttributes non-empty judgment for RowLevelOperationRuntimeGroupFiltering</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46170">[SPARK-46170]</a>: Support inject adaptive query post planner strategy rules in SparkSessionExtensions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46380">[SPARK-46380]</a>: Replacing current time prior to inline table eval</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46600">[SPARK-46600]</a>: Move shared code between SqlConf and SqlApiConf to another object</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46610">[SPARK-46610]</a>: Create table should throw exception when no value for a key in options</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45189">[SPARK-45189]</a>: Creating UnresolvedRelation from TableIdentifier should include the catalog field</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46182">[SPARK-46182]</a>: Shuffle data lost on decommissioned executor caused by race condition between lastTaskRunningTime and lastShuffleMigrationTime</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46547">[SPARK-46547]</a>: Fix deadlock issue between maintenance thread and streaming agg physical operators</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46628">[SPARK-46628]</a>: Use SPDX short identifier in <code class="language-plaintext highlighter-rouge">licenses</code> name</li>
</ul>
<h3 id="dependency-changes">Dependency Changes</h3>
<p>While being a maintenance release we did still upgrade some dependencies in this release they are:</p>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47023">[SPARK-47023]</a>: Upgrade <code class="language-plaintext highlighter-rouge">aircompressor</code> to 0.26</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-47023">[SPARK-45883]</a>: Upgrade ORC to 1.9.2</li>
</ul>
<p>You can consult JIRA for the <a href="https://s.apache.org/spark-3.5.1">detailed changes</a>.</p>
<p>We would like to acknowledge all community members for contributing patches to this release.</p>
<p>
<br/>
<a href="/news/">Spark News Archive</a>
</p>
</div>
<div class="col-12 col-md-3">
<div class="news" style="margin-bottom: 20px;">
<h5>Latest News</h5>
<ul class="list-unstyled">
<li><a href="/news/spark-3-4-3-released.html">Spark 3.4.3 released</a>
<span class="small">(Apr 18, 2024)</span></li>
<li><a href="/news/spark-3-5-1-released.html">Spark 3.5.1 released</a>
<span class="small">(Feb 23, 2024)</span></li>
<li><a href="/news/spark-3-3-4-released.html">Spark 3.3.4 released</a>
<span class="small">(Dec 16, 2023)</span></li>
<li><a href="/news/spark-3-4-2-released.html">Spark 3.4.2 released</a>
<span class="small">(Nov 30, 2023)</span></li>
</ul>
<p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
</div>
<div style="text-align:center; margin-bottom: 20px;">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
</a>
</div>
<div class="hidden-xs hidden-sm">
<a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
Download Spark
</a>
<p style="font-size: 16px; font-weight: 500; color: #555;">
Built-in Libraries:
</p>
<ul class="list-none">
<li><a href="/sql/">SQL and DataFrames</a></li>
<li><a href="/streaming/">Spark Streaming</a></li>
<li><a href="/mllib/">MLlib (machine learning)</a></li>
<li><a href="/graphx/">GraphX (graph)</a></li>
</ul>
<a href="/third-party-projects.html">Third-Party Projects</a>
</div>
</div>
</div>
<footer class="small">
<hr>
Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
All other marks mentioned may be trademarks or registered trademarks of their respective owners.
Copyright &copy; 2018 The Apache Software Foundation, Licensed under the
<a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
</footer>
</div>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js"
integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM"
crossorigin="anonymous"></script>
<script src="https://code.jquery.com/jquery.js"></script>
<script src="/js/lang-tabs.js"></script>
<script src="/js/downloads.js"></script>
</body>
</html>