Google Git
Sign in
apache / spark-website / asf-site / . / site / releases / spark-release-3-4-1.html
blob: e77e5519ca581ff97ee88876d082209080cb152c [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>
Spark Release 3.4.1 | Apache Spark
</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
<link href="/css/custom.css" rel="stylesheet">
<!-- Code highlighter CSS -->
<link href="/css/pygments-default.css" rel="stylesheet">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '40']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="global">
<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
<a class="navbar-brand" href="/">
<img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">
<ul class="navbar-nav me-auto">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Libraries
</a>
<ul class="dropdown-menu" aria-labelledby="libraries">
<li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
<li><a class="dropdown-item" href="/spark-connect/">Spark Connect</a></li>
<li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
<li><a class="dropdown-item" href="/pandas-on-spark/">pandas on Spark</a></li>
<li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
<li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
<li>
<hr class="dropdown-divider">
</li>
<li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Documentation
</a>
<ul class="dropdown-menu" aria-labelledby="documentation">
<li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
<li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
<li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Community
</a>
<ul class="dropdown-menu" aria-labelledby="community">
<li><a class="dropdown-item" href="/community.html">Mailing Lists &amp; Resources</a></li>
<li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
<li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
</li>
<li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
</li>
<li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
<li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
<li><a class="dropdown-item" href="/history.html">Project History</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Developers
</a>
<ul class="dropdown-menu" aria-labelledby="developers">
<li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
<li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
<li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
<li><a class="dropdown-item" href="/security.html">Security</a></li>
</ul>
</li>
</ul>
<ul class="navbar-nav ml-auto">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
data-bs-toggle="dropdown" aria-expanded="false">
Apache Software Foundation
</a>
<ul class="dropdown-menu" aria-labelledby="apacheFoundation">
<li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
<li><a class="dropdown-item"
href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/security/">Security</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
</ul>
</li>
</ul>
</div>
</nav>
<div class="container">
<div class="row mt-4">
<div class="col-12 col-md-9">
<h2>Spark Release 3.4.1</h2>
<p>Spark 3.4.1 is a maintenance release containing stability fixes. This release is based on the branch-3.4 maintenance branch of Spark. We strongly recommend all 3.4 users to upgrade to this stable release.</p>
<h3 id="notable-changes">Notable changes</h3>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44383">[SPARK-44383]</a>: Fix the trim logic did&#8217;t handle ASCII control characters correctly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-37829">[SPARK-37829]</a>: Dataframe.joinWith outer-join should return a null value for unmatched row</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-42078">[SPARK-42078]</a>: Add <code class="language-plaintext highlighter-rouge">CapturedException</code> to utils</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-42290">[SPARK-42290]</a>: Fix the OOM error can&#8217;t be reported when AQE on</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-42421">[SPARK-42421]</a>: Use the utils to get the switch for dynamic allocation used in local checkpoint</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-42475">[SPARK-42475]</a>: Fix PySpark connect Quickstart binder link</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-42826">[SPARK-42826]</a>: Update migration notes for pandas API on Spark</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43043">[SPARK-43043]</a>: Improve the performance of MapOutputTracker.updateMapOutput</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43050">[SPARK-43050]</a>: Fix construct aggregate expressions by replacing grouping functions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43067">[SPARK-43067]</a>: Correct the location of error class resource file in Kafka connector</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43069">[SPARK-43069]</a>: Use <code class="language-plaintext highlighter-rouge">sbt-eclipse</code> instead of <code class="language-plaintext highlighter-rouge">sbteclipse-plugin</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43071">[SPARK-43071]</a>: Support SELECT DEFAULT with ORDER BY, LIMIT, OFFSET for INSERT source relation</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43072">[SPARK-43072]</a>: Include TIMESTAMP_NTZ type in ANSI Compliance doc</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43075">[SPARK-43075]</a>: Change <code class="language-plaintext highlighter-rouge">gRPC</code> to <code class="language-plaintext highlighter-rouge">grpcio</code> when it is not installed.</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43083">[SPARK-43083]</a>: Mark <code class="language-plaintext highlighter-rouge">*StateStoreSuite</code> as <code class="language-plaintext highlighter-rouge">ExtendedSQLTest</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43085">[SPARK-43085]</a>: Support column DEFAULT assignment for multi-part table names</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43098">[SPARK-43098]</a>: Fix correctness COUNT bug when scalar subquery has group by clause</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43113">[SPARK-43113]</a>: Evaluate stream-side variables when generating code for a bound condition</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43125">[SPARK-43125]</a>: Fix Connect Server Can&#8217;t Handle Exception With Null Message</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43126">[SPARK-43126]</a>: Mark two Hive UDF expressions as stateful</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43139">[SPARK-43139]</a>: Fix incorrect column names in sql-ref-syntax-dml-insert-table.md</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43141">[SPARK-43141]</a>: Ignore generated Java files in checkstyle</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43156">[SPARK-43156]</a>: Fix <code class="language-plaintext highlighter-rouge">COUNT(*) is null</code> bug in correlated scalar subquery</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43157">[SPARK-43157]</a>: Clone InMemoryRelation cached plan to prevent cloned plan from referencing same objects</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43158">[SPARK-43158]</a>: Set upperbound of pandas version for Binder integration</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43249">[SPARK-43249]</a>: Fix missing stats for SQL Command</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43281">[SPARK-43281]</a>: Fix concurrent writer does not update file metrics</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43284">[SPARK-43284]</a>: Switch back to url-encoded strings</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43293">[SPARK-43293]</a>: <code class="language-plaintext highlighter-rouge">__qualified_access_only</code> should be ignored in normal columns</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43313">[SPARK-43313]</a>: Adding missing column DEFAULT values for MERGE INSERT actions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43336">[SPARK-43336]</a>: Casting between Timestamp and TimestampNTZ requires timezone</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43337">[SPARK-43337]</a>: Asc/desc arrow icons for sorting column does not get displayed in the table column</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43340">[SPARK-43340]</a>: Handle missing stack-trace field in eventlogs</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43342">[SPARK-43342]</a>: Revert SPARK-39006 Show a directional error message for executor PVC dynamic allocation failure</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43374">[SPARK-43374]</a>: Move protobuf-java to BSD 3-clause group and update the license copy</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43378">[SPARK-43378]</a>: Properly close stream objects in deserializeFromChunkedBuffer</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43395">[SPARK-43395]</a>: Exclude macOS tar extended metadata in make-distribution.sh</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43398">[SPARK-43398]</a>: Executor timeout should be max of idle shuffle and rdd timeout</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43404">[SPARK-43404]</a>: Skip reusing sst file for same version of RocksDB state store to avoid id mismatch error</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43414">[SPARK-43414]</a>: Fix flakiness in Kafka RDD suites due to port binding configuration issue</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43425">[SPARK-43425]</a>: Add <code class="language-plaintext highlighter-rouge">TimestampNTZType</code> to <code class="language-plaintext highlighter-rouge">ColumnarBatchRow</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43441">[SPARK-43441]</a>: <code class="language-plaintext highlighter-rouge">makeDotNode</code> should not fail when DeterministicLevel is absent</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43450">[SPARK-43450]</a>: Add more <code class="language-plaintext highlighter-rouge">_metadata</code> filter test cases</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43471">[SPARK-43471]</a>: Handle missing hadoopProperties and metricsProperties</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43483">[SPARK-43483]</a>: Adds SQL references for OFFSET clause</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43510">[SPARK-43510]</a>: Fix YarnAllocator internal state when adding running executor after processing completed containers</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43517">[SPARK-43517]</a>: Add a migration guide for namedtuple monkey patch</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43522">[SPARK-43522]</a>: Fix creating struct column name with index of array</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43527">[SPARK-43527]</a>: Fix <code class="language-plaintext highlighter-rouge">catalog.listCatalogs</code> in PySpark</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43541">[SPARK-43541]</a>: Propagate all <code class="language-plaintext highlighter-rouge">Project</code> tags in resolving of expressions and missing columns</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43547">[SPARK-43547]</a>: Update &#8220;Supported Pandas API&#8221; page to point out the proper pandas docs</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43587">[SPARK-43587]</a>: Run <code class="language-plaintext highlighter-rouge">HealthTrackerIntegrationSuite</code> in a dedicated JVM</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43589">[SPARK-43589]</a>: Fix <code class="language-plaintext highlighter-rouge">cannotBroadcastTableOverMaxTableBytesError</code> to use <code class="language-plaintext highlighter-rouge">bytesToString</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43718">[SPARK-43718]</a>: Set nullable correctly for keys in USING joins</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43719">[SPARK-43719]</a>: Handle <code class="language-plaintext highlighter-rouge">missing row.excludedInStages</code> field</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43751">[SPARK-43751]</a>: Document <code class="language-plaintext highlighter-rouge">unbase64</code> behavior change</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43758">[SPARK-43758]</a>: Update Hadoop 2 dependency manifest</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43759">[SPARK-43759]</a>: Expose TimestampNTZType in pyspark.sql.types</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43760">[SPARK-43760]</a>: Nullability of scalar subquery results</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43802">[SPARK-43802]</a>: Fix codegen for unhex and unbase64 with failOnError=true</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43894">[SPARK-43894]</a>: Fix bug in df.cache()</li>
<li>
<table>
<tbody>
<tr>
<td><a href="https://issues.apache.org/jira/browse/SPARK-43956">[SPARK-43956]</a>: Fix the bug doesn&#8217;t display column&#8217;s sql for Percentile[Cont</td>
<td>Disc]</td>
</tr>
</tbody>
</table>
</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43973">[SPARK-43973]</a>: Structured Streaming UI should display failed queries correctly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43976">[SPARK-43976]</a>: Handle the case where modifiedConfigs doesn&#8217;t exist in event logs</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44018">[SPARK-44018]</a>: Improve the hashCode and toString for some DS V2 Expression</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44038">[SPARK-44038]</a>: Update YuniKorn docs with v1.3</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44040">[SPARK-44040]</a>: Fix compute stats when AggregateExec node above QueryStageExec</li>
</ul>
<h3 id="dependency-changes">Dependency Changes</h3>
<p>While being a maintenance release we did still upgrade some dependencies in this release they are:</p>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43758">[SPARK-43758]</a>: Upgrade snappy-java to 1.1.10.0</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44070">[SPARK-44070]</a>: Upgrade snappy-java to 1.1.10.1</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43949">[SPARK-43949]</a>: Upgrade cloudpickle to 2.2.1</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44053">[SPARK-44053]</a>: Update ORC to 1.8.4</li>
</ul>
<p>You can consult JIRA for the <a href="https://s.apache.org/spark-3.4.1">detailed changes</a>.</p>
<p>We would like to acknowledge all community members for contributing patches to this release.</p>
<p>
<br/>
<a href="/news/">Spark News Archive</a>
</p>
</div>
<div class="col-12 col-md-3">
<div class="news" style="margin-bottom: 20px;">
<h5>Latest News</h5>
<ul class="list-unstyled">
<li><a href="/news/spark-3-4-3-released.html">Spark 3.4.3 released</a>
<span class="small">(Apr 18, 2024)</span></li>
<li><a href="/news/spark-3-5-1-released.html">Spark 3.5.1 released</a>
<span class="small">(Feb 23, 2024)</span></li>
<li><a href="/news/spark-3-3-4-released.html">Spark 3.3.4 released</a>
<span class="small">(Dec 16, 2023)</span></li>
<li><a href="/news/spark-3-4-2-released.html">Spark 3.4.2 released</a>
<span class="small">(Nov 30, 2023)</span></li>
</ul>
<p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
</div>
<div style="text-align:center; margin-bottom: 20px;">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
</a>
</div>
<div class="hidden-xs hidden-sm">
<a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
Download Spark
</a>
<p style="font-size: 16px; font-weight: 500; color: #555;">
Built-in Libraries:
</p>
<ul class="list-none">
<li><a href="/sql/">SQL and DataFrames</a></li>
<li><a href="/streaming/">Spark Streaming</a></li>
<li><a href="/mllib/">MLlib (machine learning)</a></li>
<li><a href="/graphx/">GraphX (graph)</a></li>
</ul>
<a href="/third-party-projects.html">Third-Party Projects</a>
</div>
</div>
</div>
<footer class="small">
<hr>
Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
All other marks mentioned may be trademarks or registered trademarks of their respective owners.
Copyright &copy; 2018 The Apache Software Foundation, Licensed under the
<a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
</footer>
</div>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js"
integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM"
crossorigin="anonymous"></script>
<script src="https://code.jquery.com/jquery.js"></script>
<script src="/js/lang-tabs.js"></script>
<script src="/js/downloads.js"></script>
</body>
</html>