site/releases/spark-release-0-6-0.html - spark-website - Git at Google

 <!DOCTYPE html>
 <html lang="en">
 <head>
   <meta charset="utf-8">
   <meta http-equiv="X-UA-Compatible" content="IE=edge">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">

   <title>
      Spark Release 0.6.0 | Apache Spark

   </title>


   <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
         integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
   <link rel="preconnect" href="https://fonts.googleapis.com">
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
   <link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
   <link href="/css/custom.css" rel="stylesheet">
   <!-- Code highlighter CSS -->
   <link href="/css/pygments-default.css" rel="stylesheet">
   <link rel="icon" href="/favicon.ico" type="image/x-icon">

   <!-- Matomo -->
   <script>
     var _paq = window._paq = window._paq || [];
     /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
     _paq.push(["disableCookies"]);
     _paq.push(['trackPageView']);
     _paq.push(['enableLinkTracking']);
     (function() {
       var u="https://analytics.apache.org/";
       _paq.push(['setTrackerUrl', u+'matomo.php']);
       _paq.push(['setSiteId', '40']);
       var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
       g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
     })();
   </script>
   <!-- End Matomo Code -->
 </head>
 <body class="global">
 <nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
   <a class="navbar-brand" href="/">
     <img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
   </a>
   <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
           aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
     <span class="navbar-toggler-icon"></span>
   </button>
   <div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">

     <ul class="navbar-nav me-auto">
       <li class="nav-item">
         <a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
       </li>
       <li class="nav-item dropdown">
         <a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
            aria-expanded="false">
           Libraries
         </a>
         <ul class="dropdown-menu" aria-labelledby="libraries">
           <li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
           <li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
           <li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
           <li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
           <li>
             <hr class="dropdown-divider">
           </li>
           <li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
         </ul>
       </li>
       <li class="nav-item dropdown">
         <a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
            aria-expanded="false">
           Documentation
         </a>
         <ul class="dropdown-menu" aria-labelledby="documentation">
           <li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
           <li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
           <li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
         </ul>
       </li>
       <li class="nav-item">
         <a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
       </li>
       <li class="nav-item dropdown">
         <a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
            aria-expanded="false">
           Community
         </a>
         <ul class="dropdown-menu" aria-labelledby="community">
           <li><a class="dropdown-item" href="/community.html">Mailing Lists &amp; Resources</a></li>
           <li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
           <li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
           </li>
           <li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
           </li>
           <li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
           <li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
           <li><a class="dropdown-item" href="/history.html">Project History</a></li>
         </ul>
       </li>
       <li class="nav-item dropdown">
         <a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
            aria-expanded="false">
           Developers
         </a>
         <ul class="dropdown-menu" aria-labelledby="developers">
           <li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
           <li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
           <li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
           <li><a class="dropdown-item" href="/security.html">Security</a></li>
         </ul>
       </li>
     </ul>
     <ul class="navbar-nav ml-auto">
       <li class="nav-item dropdown">
         <a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
            data-bs-toggle="dropdown" aria-expanded="false">
           Apache Software Foundation
         </a>
         <ul class="dropdown-menu" aria-labelledby="apacheFoundation">
           <li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
           <li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
           <li><a class="dropdown-item"
                  href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
           <li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
           <li><a class="dropdown-item" href="https://www.apache.org/security/">Security</a></li>
           <li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
         </ul>
       </li>
     </ul>
   </div>
 </nav>

 <div class="container">
   <div class="row mt-4">
     <div class="col-12 col-md-9">
       <h2>Spark Release 0.6.0</h2>


 <p>Spark 0.6.0 is a major release that brings several new features, architectural changes, and performance enhancements. The most visible additions are a standalone deploy mode, a Java API, and expanded documentation; but there are also numerous other changes under the hood, which improve performance in some cases by as much as 2x.</p>

 <p>You can download this release as either a <a href="http://github.com/downloads/mesos/spark/spark-0.6.0-sources.tar.gz">source package</a> (2 MB tar.gz) or <a href="http://github.com/downloads/mesos/spark/spark-0.6.0-prebuilt.tar.gz">prebuilt package</a> (48 MB tar.gz)</p>

 <h3>Simpler Deployment</h3>

 <p>In addition to running on Mesos, Spark now has a <a href="/docs/0.6.0/spark-standalone.html">standalone deploy mode</a> that lets you quickly launch a cluster without installing an external cluster manager. The standalone mode only needs Java installed on each machine, and Spark deployed to it.</p>

 <p>In addition, there is <a href="/docs/0.6.0/running-on-yarn.html">experimental support for running on YARN</a> (Hadoop NextGen), currently in a separate branch.</p>

 <h3>Java API</h3>

 <p>Java programmers can now use Spark through a new <a href="/docs/0.6.0/java-programming-guide.html">Java API layer</a>. This layer makes available all of Spark&#8217;s features, including parallel transformations, distributed datasets, broadcast variables, and accumulators, in a Java-friendly manner.</p>

 <h3>Expanded Documentation</h3>

 <p>Spark&#8217;s <a href="/docs/0.6.0/">documentation</a> has been expanded with a new <a href="/docs/0.6.0/quick-start.html">quick start guide</a>, additional deployment instructions, configuration guide, tuning guide, and improved <a href="/docs/0.6.0/api/core">Scaladoc</a> API documentation.</p>

 <h3>Engine Changes</h3>

 <p>Under the hood, Spark 0.6 has new, custom storage and communication layers brought in from the upcoming <a href="http://www.cs.berkeley.edu/~matei/papers/2012/hotcloud_spark_streaming.pdf">Spark Streaming</a> project. These can improve performance over past versions by as much as 2x. Specifically:</p>

 <ul>
   <li>A new communication manager using asynchronous Java NIO lets shuffle operations run faster, especially when sending large amounts of data or when jobs have many tasks.</li>
   <li>A new storage manager supports per-dataset storage level settings (e.g. whether to keep the dataset in memory, deserialized, on disk, etc, or even replicated across nodes).</li>
   <li>Spark's scheduler and control plane have been optimized to better support ultra-low-latency jobs (under 500ms) and high-throughput scheduling decisions.</li>
 </ul>

 <h3>New APIs</h3>

 <ul>
   <li>This release adds the ability to control caching strategies on a per-RDD level, so that different RDDs may be stored in memory, on disk, as serialized bytes, etc. You can choose your storage level using the <a href="/docs/0.6.0/scala-programming-guide.html#rdd-persistence"><tt>persist()</tt> method</a> on RDD.</li>
   <li>A new Accumulable class generalizes Accumulators for the case when the type being accumulated is not the same as the types of elements being added (e.g. you wish to accumulate a collection, such as a Set, by adding individual elements).</li>
   <li>You can now dynamically add files or JARs that should be shipped to your workers with <tt>SparkContext.addFile/Jar</tt>.</li>
   <li>More Spark operators (e.g. joins) support custom partitioners.</li>
 </ul>

 <h3>Enhanced Debugging</h3>

 <p>Spark&#8217;s log now prints which operation in your program each RDD and job described in your logs belongs to, making it easier to tie back to which parts of your code experience problems.</p>

 <h3>Maven Artifacts</h3>

 <p>Spark is now available in Maven Central, making it easier to link into your programs without having to build it as a JAR. Use the following Maven identifiers to add it to a project:</p>

 <ul>
   <li>groupId: org.spark-project</li>
   <li>artifactId: spark-core_2.9.2</li>
   <li>version: 0.6.0</li>
 </ul>

 <h3>Compatibility</h3>

 <p>This release is source-compatible with Spark 0.5 programs, but you will need to recompile them against 0.6. In addition,  the configuration for caching has changed: instead of having a <tt>spark.cache.class</tt> parameter that sets one caching strategy for all RDDs, you can now set a <a href="/docs/0.6.0/scala-programming-guide.html#rdd-persistence">per-RDD storage level</a>. Spark will warn if you try to set <tt>spark.cache.class</tt>.</p>

 <h3>Credits</h3>

 <p>Spark 0.6 was the work of a large set of new contributors from Berkeley and outside.</p>

 <ul>
   <li>Tathagata Das contributed the new communication layer, and parts of the storage layer.</li>
   <li>Haoyuan Li contributed the new storage manager.</li>
   <li>Denny Britz contributed the YARN deploy mode, key aspects of the standalone one, and several other features.</li>
   <li>Andy Konwinski contributed the revamped documentation site, Maven publishing, and several API docs.</li>
   <li>Josh Rosen contributed the Java API, and several bug fixes.</li>
   <li>Patrick Wendell contributed the enhanced debugging feature and helped with testing and documentation.</li>
   <li>Reynold Xin contributed numerous bug and performance fixes.</li>
   <li>Imran Rashid contributed the new Accumulable class.</li>
   <li>Harvey Feng contributed improvements to shuffle operations.</li>
   <li>Shivaram Venkataraman improved Spark's memory estimation and wrote a memory tuning guide.</li>
   <li>Ravi Pandya contributed Spark run scripts for Windows.
   </li><li>Mosharaf Chowdhury provided several fixes to broadcast.</li>
   <li>Henry Milner pointed out several bugs in sampling algorithms.</li>
   <li>Ray Racine provided improvements to the EC2 scripts.</li>
   <li>Paul Ruan and Bill Zhao helped with testing.</li>
 </ul>

 <p style="padding-top:5px;">Thanks also to all the Spark users who have diligently suggested features or reported bugs.</p>


 <p>
 <br/>
 <a href="/news/">Spark News Archive</a>
 </p>

     </div>
     <div class="col-12 col-md-3">
       <div class="news" style="margin-bottom: 20px;">
         <h5>Latest News</h5>
         <ul class="list-unstyled">

           <li><a href="/news/spark-3-5-1-released.html">Spark 3.5.1 released</a>
             <span class="small">(Feb 23, 2024)</span></li>

           <li><a href="/news/spark-3-3-4-released.html">Spark 3.3.4 released</a>
             <span class="small">(Dec 16, 2023)</span></li>

           <li><a href="/news/spark-3-4-2-released.html">Spark 3.4.2 released</a>
             <span class="small">(Nov 30, 2023)</span></li>

           <li><a href="/news/spark-3-5-0-released.html">Spark 3.5.0 released</a>
             <span class="small">(Sep 13, 2023)</span></li>

         </ul>
         <p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
       </div>
       <div style="text-align:center; margin-bottom: 20px;">
         <a href="https://www.apache.org/events/current-event.html">
           <img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
         </a>
       </div>
       <div class="hidden-xs hidden-sm">
         <a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
           Download Spark
         </a>
         <p style="font-size: 16px; font-weight: 500; color: #555;">
           Built-in Libraries:
         </p>
         <ul class="list-none">
           <li><a href="/sql/">SQL and DataFrames</a></li>
           <li><a href="/streaming/">Spark Streaming</a></li>
           <li><a href="/mllib/">MLlib (machine learning)</a></li>
           <li><a href="/graphx/">GraphX (graph)</a></li>
         </ul>
         <a href="/third-party-projects.html">Third-Party Projects</a>
       </div>
     </div>
   </div>


   <footer class="small">
     <hr>
     Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
     trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
     See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
     All other marks mentioned may be trademarks or registered trademarks of their respective owners.
     Copyright &copy; 2018 The Apache Software Foundation, Licensed under the
     <a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
   </footer>
 </div>

 <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js"
         integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM"
         crossorigin="anonymous"></script>
 <script src="https://code.jquery.com/jquery.js"></script>
 <script src="/js/lang-tabs.js"></script>
 <script src="/js/downloads.js"></script>
 </body>
 </html>
	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="utf-8">
	<meta http-equiv="X-UA-Compatible" content="IE=edge">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">

	<title>
	Spark Release 0.6.0 \| Apache Spark

	</title>





	<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
	integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
	<link rel="preconnect" href="https://fonts.googleapis.com">
	<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
	<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
	<link href="/css/custom.css" rel="stylesheet">
	<!-- Code highlighter CSS -->
	<link href="/css/pygments-default.css" rel="stylesheet">
	<link rel="icon" href="/favicon.ico" type="image/x-icon">

	<!-- Matomo -->
	<script>
	var _paq = window._paq = window._paq \|\| [];
	/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
	_paq.push(["disableCookies"]);
	_paq.push(['trackPageView']);
	_paq.push(['enableLinkTracking']);
	(function() {
	var u="https://analytics.apache.org/";
	_paq.push(['setTrackerUrl', u+'matomo.php']);
	_paq.push(['setSiteId', '40']);
	var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
	g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
	})();
	</script>
	<!-- End Matomo Code -->
	</head>
	<body class="global">
	<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4" style="background: #1D6890;">
	<a class="navbar-brand" href="/">
	<img src="/images/spark-logo-rev.svg" alt="" width="141" height="72">
	</a>
	<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarContent"
	aria-controls="navbarContent" aria-expanded="false" aria-label="Toggle navigation">
	<span class="navbar-toggler-icon"></span>
	</button>
	<div class="collapse navbar-collapse col-md-12 col-lg-auto pt-4" id="navbarContent">

	<ul class="navbar-nav me-auto">
	<li class="nav-item">
	<a class="nav-link active" aria-current="page" href="/downloads.html">Download</a>
	</li>
	<li class="nav-item dropdown">
	<a class="nav-link dropdown-toggle" href="#" id="libraries" role="button" data-bs-toggle="dropdown"
	aria-expanded="false">
	Libraries
	</a>
	<ul class="dropdown-menu" aria-labelledby="libraries">
	<li><a class="dropdown-item" href="/sql/">SQL and DataFrames</a></li>
	<li><a class="dropdown-item" href="/streaming/">Spark Streaming</a></li>
	<li><a class="dropdown-item" href="/mllib/">MLlib (machine learning)</a></li>
	<li><a class="dropdown-item" href="/graphx/">GraphX (graph)</a></li>
	<li>
	<hr class="dropdown-divider">
	</li>
	<li><a class="dropdown-item" href="/third-party-projects.html">Third-Party Projects</a></li>
	</ul>
	</li>
	<li class="nav-item dropdown">
	<a class="nav-link dropdown-toggle" href="#" id="documentation" role="button" data-bs-toggle="dropdown"
	aria-expanded="false">
	Documentation
	</a>
	<ul class="dropdown-menu" aria-labelledby="documentation">
	<li><a class="dropdown-item" href="/docs/latest/">Latest Release</a></li>
	<li><a class="dropdown-item" href="/documentation.html">Older Versions and Other Resources</a></li>
	<li><a class="dropdown-item" href="/faq.html">Frequently Asked Questions</a></li>
	</ul>
	</li>
	<li class="nav-item">
	<a class="nav-link active" aria-current="page" href="/examples.html">Examples</a>
	</li>
	<li class="nav-item dropdown">
	<a class="nav-link dropdown-toggle" href="#" id="community" role="button" data-bs-toggle="dropdown"
	aria-expanded="false">
	Community
	</a>
	<ul class="dropdown-menu" aria-labelledby="community">
	<li><a class="dropdown-item" href="/community.html">Mailing Lists & Resources</a></li>
	<li><a class="dropdown-item" href="/contributing.html">Contributing to Spark</a></li>
	<li><a class="dropdown-item" href="/improvement-proposals.html">Improvement Proposals (SPIP)</a>
	</li>
	<li><a class="dropdown-item" href="https://issues.apache.org/jira/browse/SPARK">Issue Tracker</a>
	</li>
	<li><a class="dropdown-item" href="/powered-by.html">Powered By</a></li>
	<li><a class="dropdown-item" href="/committers.html">Project Committers</a></li>
	<li><a class="dropdown-item" href="/history.html">Project History</a></li>
	</ul>
	</li>
	<li class="nav-item dropdown">
	<a class="nav-link dropdown-toggle" href="#" id="developers" role="button" data-bs-toggle="dropdown"
	aria-expanded="false">
	Developers
	</a>
	<ul class="dropdown-menu" aria-labelledby="developers">
	<li><a class="dropdown-item" href="/developer-tools.html">Useful Developer Tools</a></li>
	<li><a class="dropdown-item" href="/versioning-policy.html">Versioning Policy</a></li>
	<li><a class="dropdown-item" href="/release-process.html">Release Process</a></li>
	<li><a class="dropdown-item" href="/security.html">Security</a></li>
	</ul>
	</li>
	</ul>
	<ul class="navbar-nav ml-auto">
	<li class="nav-item dropdown">
	<a class="nav-link dropdown-toggle" href="#" id="apacheFoundation" role="button"
	data-bs-toggle="dropdown" aria-expanded="false">
	Apache Software Foundation
	</a>
	<ul class="dropdown-menu" aria-labelledby="apacheFoundation">
	<li><a class="dropdown-item" href="https://www.apache.org/">Apache Homepage</a></li>
	<li><a class="dropdown-item" href="https://www.apache.org/licenses/">License</a></li>
	<li><a class="dropdown-item"
	href="https://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
	<li><a class="dropdown-item" href="https://www.apache.org/foundation/thanks.html">Thanks</a></li>
	<li><a class="dropdown-item" href="https://www.apache.org/security/">Security</a></li>
	<li><a class="dropdown-item" href="https://www.apache.org/events/current-event">Event</a></li>
	</ul>
	</li>
	</ul>
	</div>
	</nav>

	<div class="container">
	<div class="row mt-4">
	<div class="col-12 col-md-9">
	<h2>Spark Release 0.6.0</h2>


	<p>Spark 0.6.0 is a major release that brings several new features, architectural changes, and performance enhancements. The most visible additions are a standalone deploy mode, a Java API, and expanded documentation; but there are also numerous other changes under the hood, which improve performance in some cases by as much as 2x.</p>

	<p>You can download this release as either a <a href="http://github.com/downloads/mesos/spark/spark-0.6.0-sources.tar.gz">source package</a> (2 MB tar.gz) or <a href="http://github.com/downloads/mesos/spark/spark-0.6.0-prebuilt.tar.gz">prebuilt package</a> (48 MB tar.gz)</p>

	<h3>Simpler Deployment</h3>

	<p>In addition to running on Mesos, Spark now has a <a href="/docs/0.6.0/spark-standalone.html">standalone deploy mode</a> that lets you quickly launch a cluster without installing an external cluster manager. The standalone mode only needs Java installed on each machine, and Spark deployed to it.</p>

	<p>In addition, there is <a href="/docs/0.6.0/running-on-yarn.html">experimental support for running on YARN</a> (Hadoop NextGen), currently in a separate branch.</p>

	<h3>Java API</h3>

	<p>Java programmers can now use Spark through a new <a href="/docs/0.6.0/java-programming-guide.html">Java API layer</a>. This layer makes available all of Spark’s features, including parallel transformations, distributed datasets, broadcast variables, and accumulators, in a Java-friendly manner.</p>

	<h3>Expanded Documentation</h3>

	<p>Spark’s <a href="/docs/0.6.0/">documentation</a> has been expanded with a new <a href="/docs/0.6.0/quick-start.html">quick start guide</a>, additional deployment instructions, configuration guide, tuning guide, and improved <a href="/docs/0.6.0/api/core">Scaladoc</a> API documentation.</p>

	<h3>Engine Changes</h3>

	<p>Under the hood, Spark 0.6 has new, custom storage and communication layers brought in from the upcoming <a href="http://www.cs.berkeley.edu/~matei/papers/2012/hotcloud_spark_streaming.pdf">Spark Streaming</a> project. These can improve performance over past versions by as much as 2x. Specifically:</p>

	<ul>
	<li>A new communication manager using asynchronous Java NIO lets shuffle operations run faster, especially when sending large amounts of data or when jobs have many tasks.</li>
	<li>A new storage manager supports per-dataset storage level settings (e.g. whether to keep the dataset in memory, deserialized, on disk, etc, or even replicated across nodes).</li>
	<li>Spark's scheduler and control plane have been optimized to better support ultra-low-latency jobs (under 500ms) and high-throughput scheduling decisions.</li>
	</ul>

	<h3>New APIs</h3>

	<ul>
	<li>This release adds the ability to control caching strategies on a per-RDD level, so that different RDDs may be stored in memory, on disk, as serialized bytes, etc. You can choose your storage level using the <a href="/docs/0.6.0/scala-programming-guide.html#rdd-persistence"><tt>persist()</tt> method</a> on RDD.</li>
	<li>A new Accumulable class generalizes Accumulators for the case when the type being accumulated is not the same as the types of elements being added (e.g. you wish to accumulate a collection, such as a Set, by adding individual elements).</li>
	<li>You can now dynamically add files or JARs that should be shipped to your workers with <tt>SparkContext.addFile/Jar</tt>.</li>
	<li>More Spark operators (e.g. joins) support custom partitioners.</li>
	</ul>

	<h3>Enhanced Debugging</h3>

	<p>Spark’s log now prints which operation in your program each RDD and job described in your logs belongs to, making it easier to tie back to which parts of your code experience problems.</p>

	<h3>Maven Artifacts</h3>

	<p>Spark is now available in Maven Central, making it easier to link into your programs without having to build it as a JAR. Use the following Maven identifiers to add it to a project:</p>

	<ul>
	<li>groupId: org.spark-project</li>
	<li>artifactId: spark-core_2.9.2</li>
	<li>version: 0.6.0</li>
	</ul>

	<h3>Compatibility</h3>

	<p>This release is source-compatible with Spark 0.5 programs, but you will need to recompile them against 0.6. In addition, the configuration for caching has changed: instead of having a <tt>spark.cache.class</tt> parameter that sets one caching strategy for all RDDs, you can now set a <a href="/docs/0.6.0/scala-programming-guide.html#rdd-persistence">per-RDD storage level</a>. Spark will warn if you try to set <tt>spark.cache.class</tt>.</p>

	<h3>Credits</h3>

	<p>Spark 0.6 was the work of a large set of new contributors from Berkeley and outside.</p>

	<ul>
	<li>Tathagata Das contributed the new communication layer, and parts of the storage layer.</li>
	<li>Haoyuan Li contributed the new storage manager.</li>
	<li>Denny Britz contributed the YARN deploy mode, key aspects of the standalone one, and several other features.</li>
	<li>Andy Konwinski contributed the revamped documentation site, Maven publishing, and several API docs.</li>
	<li>Josh Rosen contributed the Java API, and several bug fixes.</li>
	<li>Patrick Wendell contributed the enhanced debugging feature and helped with testing and documentation.</li>
	<li>Reynold Xin contributed numerous bug and performance fixes.</li>
	<li>Imran Rashid contributed the new Accumulable class.</li>
	<li>Harvey Feng contributed improvements to shuffle operations.</li>
	<li>Shivaram Venkataraman improved Spark's memory estimation and wrote a memory tuning guide.</li>
	<li>Ravi Pandya contributed Spark run scripts for Windows.
	</li><li>Mosharaf Chowdhury provided several fixes to broadcast.</li>
	<li>Henry Milner pointed out several bugs in sampling algorithms.</li>
	<li>Ray Racine provided improvements to the EC2 scripts.</li>
	<li>Paul Ruan and Bill Zhao helped with testing.</li>
	</ul>

	<p style="padding-top:5px;">Thanks also to all the Spark users who have diligently suggested features or reported bugs.</p>


	<p>
	<br/>
	<a href="/news/">Spark News Archive</a>
	</p>

	</div>
	<div class="col-12 col-md-3">
	<div class="news" style="margin-bottom: 20px;">
	<h5>Latest News</h5>
	<ul class="list-unstyled">

	<li><a href="/news/spark-3-5-1-released.html">Spark 3.5.1 released</a>
	<span class="small">(Feb 23, 2024)</span></li>

	<li><a href="/news/spark-3-3-4-released.html">Spark 3.3.4 released</a>
	<span class="small">(Dec 16, 2023)</span></li>

	<li><a href="/news/spark-3-4-2-released.html">Spark 3.4.2 released</a>
	<span class="small">(Nov 30, 2023)</span></li>

	<li><a href="/news/spark-3-5-0-released.html">Spark 3.5.0 released</a>
	<span class="small">(Sep 13, 2023)</span></li>

	</ul>
	<p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
	</div>
	<div style="text-align:center; margin-bottom: 20px;">
	<a href="https://www.apache.org/events/current-event.html">
	<img src="https://www.apache.org/events/current-event-234x60.png" style="max-width: 100%;"/>
	</a>
	</div>
	<div class="hidden-xs hidden-sm">
	<a href="/downloads.html" class="btn btn-cta btn-lg d-grid" style="margin-bottom: 30px;">
	Download Spark
	</a>
	<p style="font-size: 16px; font-weight: 500; color: #555;">
	Built-in Libraries:
	</p>
	<ul class="list-none">
	<li><a href="/sql/">SQL and DataFrames</a></li>
	<li><a href="/streaming/">Spark Streaming</a></li>
	<li><a href="/mllib/">MLlib (machine learning)</a></li>
	<li><a href="/graphx/">GraphX (graph)</a></li>
	</ul>
	<a href="/third-party-projects.html">Third-Party Projects</a>
	</div>
	</div>
	</div>



	<footer class="small">
	<hr>
	Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered
	trademarks or trademarks of The Apache Software Foundation in the United States and other countries.
	See guidance on use of Apache Spark <a href="/trademarks.html">trademarks</a>.
	All other marks mentioned may be trademarks or registered trademarks of their respective owners.
	Copyright © 2018 The Apache Software Foundation, Licensed under the
	<a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.
	</footer>
	</div>

	<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js"
	integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM"
	crossorigin="anonymous"></script>
	<script src="https://code.jquery.com/jquery.js"></script>
	<script src="/js/lang-tabs.js"></script>
	<script src="/js/downloads.js"></script>
	</body>
	</html>