blob: 8c1f68015a869a34571c77c4bda9b3180547e148 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>
Spark Release 3.4.2 | Apache Spark
</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
<link href="/css/custom.css" rel="stylesheet">
<!-- Code highlighter CSS -->
<link href="/css/pygments-default.css" rel="stylesheet">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '40']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body class="global">
<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
<a class="navbar-brand" href="/">
<img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">
<ul class="navbar-nav me-auto">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Libraries
</a>
<ul class="dropdown-menu" aria-labelledby="libraries">
<li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
<li><a class="dropdown-item" href="/spark-connect/">Spark Connect</a></li>
<li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
<li><a class="dropdown-item" href="/pandas-on-spark/">pandas on Spark</a></li>
<li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
<li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
<li>
<hr class="dropdown-divider">
</li>
<li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Documentation
</a>
<ul class="dropdown-menu" aria-labelledby="documentation">
<li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
<li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
<li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Community
</a>
<ul class="dropdown-menu" aria-labelledby="community">
<li><a class="dropdown-item" href="/community.html">Mailing Lists &amp; Resources</a></li>
<li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
<li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
</li>
<li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
</li>
<li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
<li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
<li><a class="dropdown-item" href="/history.html">Project History</a></li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
aria-expanded="false">
Developers
</a>
<ul class="dropdown-menu" aria-labelledby="developers">
<li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
<li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
<li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
<li><a class="dropdown-item" href="/security.html">Security</a></li>
</ul>
</li>
</ul>
<ul class="navbar-nav ml-auto">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
data-bs-toggle="dropdown" aria-expanded="false">
Apache Software Foundation
</a>
<ul class="dropdown-menu" aria-labelledby="apacheFoundation">
<li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
<li><a class="dropdown-item"
href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/security/">Security</a></li>
<li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
</ul>
</li>
</ul>
</div>
</nav>
<div class="container">
<div class="row mt-4">
<div class="col-12 col-md-9">
<h2>Spark Release 3.4.2</h2>
<p>Spark 3.4.2 is a maintenance release containing security and correctness fixes. This release is based on the branch-3.4 maintenance branch of Spark. We strongly recommend all 3.4 users to upgrade to this stable release.</p>
<h3 id="notable-changes">Notable changes</h3>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-42784">[SPARK-42784]</a>: should still create subDir when the number of subDir in merge dir is less than conf</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43203">[SPARK-43203]</a>: Fix DROP table behavior in session catalog</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-43393">[SPARK-43393]</a>: Address sequence expression overflow bug</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44040">[SPARK-44040]</a>: Fix compute stats when AggregateExec node above QueryStageExec</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44079">[SPARK-44079]</a>: Fix <code class="language-plaintext highlighter-rouge">ArrayIndexOutOfBoundsException</code> when parse array as struct using PERMISSIVE mode with corrupt record</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44134">[SPARK-44134]</a>: Fix setting resources (GPU/FPGA) to 0 when they are set in spark-defaults.conf</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44136">[SPARK-44136]</a>: Fixed an issue that StateManager may get materialized in executor instead of driver in FlatMapGroupsWithStateExec</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44142">[SPARK-44142]</a>: Replace type with tpe in utility to convert python types to spark types</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44180">[SPARK-44180]</a>: DistributionAndOrderingUtils should apply ResolveTimeZone</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44206">[SPARK-44206]</a>: DataSet.selectExpr scope Session.active</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44215">[SPARK-44215]</a>: If num chunks are 0, then server should throw a RuntimeException</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44241">[SPARK-44241]</a>: Mistakenly set io.connectionTimeout/connectionCreationTimeout to zero or negative will cause incessant executor cons/destructions</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44251">[SPARK-44251]</a>: Set nullable correctly on coalesced join key in full outer USING join</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44313">[SPARK-44313]</a>: Fix generated column expression validation when there is a char/varchar column in the schema</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44391">[SPARK-44391]</a>: Check the number of argument types in <code class="language-plaintext highlighter-rouge">InvokeLike</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44464">[SPARK-44464]</a>: Fix applyInPandasWithStatePythonRunner to output rows that have Null as first column value</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44479">[SPARK-44479]</a>: Fix protobuf conversion from an empty struct type</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44547">[SPARK-44547]</a>: Ignore fallback storage for cached RDD migration</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44581">[SPARK-44581]</a>: Fix the bug that ShutdownHookManager gets wrong UGI from SecurityManager of ApplicationMaster</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44588">[SPARK-44588]</a>: Fix double encryption issue for migrated shuffle blocks</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44630">[SPARK-44630]</a>: Revert &#8220;[SPARK-43043] Improve the performance of MapOutputTracker.updateMapOutput&#8221;</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44634">[SPARK-44634]</a>: Encoders.bean does no longer support nested beans with type arguments</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44641">[SPARK-44641]</a>: Incorrect result in certain scenarios when SPJ is not triggered</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44653">[SPARK-44653]</a>: Non-trivial DataFrame unions should not break caching</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44657">[SPARK-44657]</a>: Fix incorrect limit handling in ArrowBatchWithSchemaIterator and config parsing of CONNECT_GRPC_ARROW_MAX_BATCH_SIZE</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44805">[SPARK-44805]</a>: getBytes/getShorts/getInts/etc. should work in a column vector that has a dictionary</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44840">[SPARK-44840]</a>: Make <code class="language-plaintext highlighter-rouge">array_insert()</code> 1-based for negative indexes</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44846">[SPARK-44846]</a>: Convert the lower redundant Aggregate to Project in RemoveRedundantAggregates</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44854">[SPARK-44854]</a>: Python timedelta to DayTimeIntervalType edge case bug</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44857">[SPARK-44857]</a>: Fix <code class="language-plaintext highlighter-rouge">getBaseURI</code> error in Spark Worker LogPage UI buttons</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44859">[SPARK-44859]</a>: Fix incorrect property name in structured streaming doc</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44871">[SPARK-44871]</a>: Fix percentile_disc behaviour</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44910">[SPARK-44910]</a>: Encoders.bean does not support superclasses with generic type arguments</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44920">[SPARK-44920]</a>: Use await() instead of awaitUninterruptibly() in TransportClientFactory.createClient()</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44925">[SPARK-44925]</a>: K8s default service token file should not be materialized into token</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44935">[SPARK-44935]</a>: Fix <code class="language-plaintext highlighter-rouge">RELEASE</code> file to have the correct information in Docker images if exists</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44937">[SPARK-44937]</a>: Mark connection as timedOut in TransportClient.close</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44940">[SPARK-44940]</a>: Improve performance of JSON parsing when &#8220;spark.sql.json.enablePartialResults&#8221; is enabled</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44973">[SPARK-44973]</a>: Fix <code class="language-plaintext highlighter-rouge">ArrayIndexOutOfBoundsException</code> in <code class="language-plaintext highlighter-rouge">conv()</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44990">[SPARK-44990]</a>: Reduce the frequency of get <code class="language-plaintext highlighter-rouge">spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv</code></li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45054">[SPARK-45054]</a>: HiveExternalCatalog.listPartitions should restore partition statistics</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45057">[SPARK-45057]</a>: Avoid acquire read lock when keepReadLock is false</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45071">[SPARK-45071]</a>: Optimize the processing speed of <code class="language-plaintext highlighter-rouge">BinaryArithmetic#dataType</code> when processing multi-column data</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45075">[SPARK-45075]</a>: Fix alter table with invalid default value will not report error</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45078">[SPARK-45078]</a>: Fix <code class="language-plaintext highlighter-rouge">array_insert</code> ImplicitCastInputTypes not work</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45079">[SPARK-45079]</a>: Fix an internal error from <code class="language-plaintext highlighter-rouge">percentile_approx()</code>on <code class="language-plaintext highlighter-rouge">NULL</code> accuracy</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45081">[SPARK-45081]</a>: Encoders.bean does no longer work with read-only properties</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45100">[SPARK-45100]</a>: Fix an internal error from <code class="language-plaintext highlighter-rouge">reflect()</code>on <code class="language-plaintext highlighter-rouge">NULL</code> class and method</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45109">[SPARK-45109]</a>: Fix log function in Connect</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45187">[SPARK-45187]</a>: Fix <code class="language-plaintext highlighter-rouge">WorkerPage</code> to use the same pattern for <code class="language-plaintext highlighter-rouge">logPage</code> urls</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45227">[SPARK-45227]</a>: Fix a subtle thread-safety issue with CoarseGrainedExecutorBackend</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45282">[SPARK-45282]</a>: Correctness issue in AQE with InMemoryTableScanExec</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45389">[SPARK-45389]</a>: Correct MetaException matching rule on getting partition metadata</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45430">[SPARK-45430]</a>: Fix for FramelessOffsetWindowFunction when IGNORE NULLS and offset &gt; rowCount</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45433">[SPARK-45433]</a>: Fix CSV/JSON schema inference when timestamps do not match specified timestampFormat</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45473">[SPARK-45473]</a>: Fix incorrect error message for RoundBase</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45508">[SPARK-45508]</a>: Add &#8220;&#8211;add-opens=java.base/jdk.internal.ref=ALL-UNNAMED&#8221; so Platform can access Cleaner on Java 9+</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45592">[SPARK-45592]</a>: Correctness issue in AQE with InMemoryTableScanExec</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45604">[SPARK-45604]</a>: Add LogicalType checking on INT64 -&gt; DateTime conversion on Parquet Vectorized Reader</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45652">[SPARK-45652]</a>: SPJ: Handle empty input partitions after dynamic filtering</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45670">[SPARK-45670]</a>: SparkSubmit does not support <code class="language-plaintext highlighter-rouge">--total-executor-cores</code> when deploying on K8s</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45678">[SPARK-45678]</a>: Cover BufferReleasingInputStream.available/reset under tryOrFetchFailedException</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45749">[SPARK-45749]</a>: Fix <code class="language-plaintext highlighter-rouge">Spark History Server</code> to sort <code class="language-plaintext highlighter-rouge">Duration</code> column properly</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45786">[SPARK-45786]</a>: Fix inaccurate Decimal multiplication and division results</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45814">[SPARK-45814]</a>: Make ArrowConverters.createEmptyArrowBatch call close() to avoid memory leak</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45882">[SPARK-45882]</a>: BroadcastHashJoinExec propagate partitioning should respect CoalescedHashPartitioning</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45896">[SPARK-45896]</a>: Construct <code class="language-plaintext highlighter-rouge">ValidateExternalType</code> with the correct expected type</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45920">[SPARK-45920]</a>: group by ordinal should be idempotent</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46006">[SPARK-46006]</a>: YarnAllocator miss clean targetNumExecutorsPerResourceProfileId after YarnSchedulerBackend call stop</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46012">[SPARK-46012]</a>: EventLogFileReader should not read rolling logs if app status file is missing</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46062">[SPARK-46062]</a>: Sync the isStreaming flag between CTE definition and reference</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-46064">[SPARK-46064]</a>: Move out EliminateEventTimeWatermark to the analyzer and change to only take effect on resolved child</li>
</ul>
<h3 id="dependency-changes">Dependency Changes</h3>
<p>While being a maintenance release we did still upgrade some dependencies in this release they are:</p>
<ul>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44415">[SPARK-44415]</a>: Upgrade snappy-java to 1.1.10.2</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-44513">[SPARK-44513]</a>: Upgrade snappy-java to 1.1.10.3</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45103">[SPARK-45103]</a>: Update ORC to 1.8.5</li>
<li><a href="https://issues.apache.org/jira/browse/SPARK-45884">[SPARK-45884]</a>: Update ORC to 1.8.6</li>
</ul>
<p>You can consult JIRA for the <a href="https://s.apache.org/spark-3.4.2">detailed changes</a>.</p>
<p>We would like to acknowledge all community members for contributing patches to this release.</p>
<p>
<br/>
<a href="/news/">Spark News Archive</a>
</p>
</div>
<div class="col-12 col-md-3">
<div class="news" style="margin-bottom: 20px;">
<h5>Latest News</h5>
<ul class="list-unstyled">
<li><a href="/news/spark-3-4-3-released.html">Spark 3.4.3 released</a>
<span class="small">(Apr 18, 2024)</span></li>
<li><a href="/news/spark-3-5-1-released.html">Spark 3.5.1 released</a>
<span class="small">(Feb 23, 2024)</span></li>
<li><a href="/news/spark-3-3-4-released.html">Spark 3.3.4 released</a>
<span class="small">(Dec 16, 2023)</span></li>
<li><a href="/news/spark-3-4-2-released.html">Spark 3.4.2 released</a>
<span class="small">(Nov 30, 2023)</span></li>
</ul>
<p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
</div>
<div style="text-align:center; margin-bottom: 20px;">
<a href="https://www.apache.org/events/current-event.html">
<img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
</a>
</div>
<div class="hidden-xs hidden-sm">
<a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
Download Spark
</a>
<p style="font-size: 16px; font-weight: 500; color: #555;">
Built-in Libraries:
</p>
<ul class="list-none">
<li><a href="/sql/">SQL and DataFrames</a></li>
<li><a href="/streaming/">Spark Streaming</a></li>
<li><a href="/mllib/">MLlib (machine learning)</a></li>
<li><a href="/graphx/">GraphX (graph)</a></li>
</ul>
<a href="/third-party-projects.html">Third-Party Projects</a>
</div>
</div>
</div>
<footer class="small">
<hr>
Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
All other marks mentioned may be trademarks or registered trademarks of their respective owners.
Copyright &copy; 2018 The Apache Software Foundation, Licensed under the
<a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
</footer>
</div>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js"
integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM"
crossorigin="anonymous"></script>
<script src="https://code.jquery.com/jquery.js"></script>
<script src="/js/lang-tabs.js"></script>
<script src="/js/downloads.js"></script>
</body>
</html>