site/docs/4.1.0-preview1/mllib-guide.html - spark-website - Git at Google



 <!DOCTYPE html>
 <html class="no-js">
     <head>
         <meta charset="utf-8">
         <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
         <meta name="viewport" content="width=device-width, initial-scale=1.0">

         <title>MLlib: RDD-based API - Spark 4.1.0-preview1 Documentation</title>


         <link rel="stylesheet" href="css/bootstrap.min.css">
         <link rel="preconnect" href="https://fonts.googleapis.com">
         <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
         <link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
         <link href="css/custom.css" rel="stylesheet">
         <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>

         <link rel="stylesheet" href="css/pygments-default.css">
         <link rel="stylesheet" href="css/docsearch.min.css" />
         <link rel="stylesheet" href="css/docsearch.css">


         <!-- Matomo -->
         <script>
             var _paq = window._paq = window._paq || [];
             /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
             _paq.push(["disableCookies"]);
             _paq.push(['trackPageView']);
             _paq.push(['enableLinkTracking']);
             (function() {
               var u="https://analytics.apache.org/";
               _paq.push(['setTrackerUrl', u+'matomo.php']);
               _paq.push(['setSiteId', '40']);
               var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
               g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
             })();
         </script>
         <!-- End Matomo Code -->


     </head>
     <body class="global">
         <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->
         <nav class="navbar navbar-expand-lg navbar-dark p-0 px-4 fixed-top" style="background: #1d6890;" id="topbar">
             <div class="navbar-brand"><a href="index.html">
                 <img src="https://spark.apache.org/images/spark-logo-rev.svg" width="141" height="72"/></a><span class="version">4.1.0-preview1</span>
             </div>
             <button class="navbar-toggler" type="button" data-toggle="collapse"
                     data-target="#navbarCollapse" aria-controls="navbarCollapse"
                     aria-expanded="false" aria-label="Toggle navigation">
                 <span class="navbar-toggler-icon"></span>
             </button>
             <div class="collapse navbar-collapse" id="navbarCollapse">
                 <ul class="navbar-nav me-auto">
                     <li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a>
                         <div class="dropdown-menu" aria-labelledby="navbarQuickStart">
                             <a class="dropdown-item" href="quick-start.html">Quick Start</a>
                             <a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a>
                             <a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a>
                             <a class="dropdown-item" href="streaming/index.html">Structured Streaming</a>
                             <a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a>
                             <a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a>
                             <a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a>
                             <a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a>
                             <a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a>
                             <a class="dropdown-item" href="declarative-pipelines-programming-guide.html">Declarative Pipelines</a>
                         </div>
                     </li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a>
                         <div class="dropdown-menu" aria-labelledby="navbarAPIDocs">
                             <a class="dropdown-item" href="api/python/index.html">Python</a>
                             <a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a>
                             <a class="dropdown-item" href="api/java/index.html">Java</a>
                             <a class="dropdown-item" href="api/R/index.html">R</a>
                             <a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a>
                         </div>
                     </li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a>
                         <div class="dropdown-menu" aria-labelledby="navbarDeploying">
                             <a class="dropdown-item" href="cluster-overview.html">Overview</a>
                             <a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a>
                             <div class="dropdown-divider"></div>
                             <a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a>
                             <a class="dropdown-item" href="running-on-yarn.html">YARN</a>
                             <a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a>
                         </div>
                     </li>

                     <li class="nav-item dropdown">
                         <a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
                         <div class="dropdown-menu" aria-labelledby="navbarMore">
                             <a class="dropdown-item" href="configuration.html">Configuration</a>
                             <a class="dropdown-item" href="monitoring.html">Monitoring</a>
                             <a class="dropdown-item" href="tuning.html">Tuning Guide</a>
                             <a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a>
                             <a class="dropdown-item" href="security.html">Security</a>
                             <a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a>
                             <a class="dropdown-item" href="migration-guide.html">Migration Guide</a>
                             <div class="dropdown-divider"></div>
                             <a class="dropdown-item" href="building-spark.html">Building Spark</a>
                             <a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a>
                             <a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a>
                         </div>
                     </li>

                     <li class="nav-item">
                         <input type="text" id="docsearch-input" placeholder="Search the docs…">
                     </li>
                 </ul>
                 <!--<span class="navbar-text navbar-right"><span class="version-text">v4.1.0-preview1</span></span>-->
             </div>
         </nav>


         <div class="container">


                     <div class="left-menu-wrapper">
     <div class="left-menu">
         <h3><a href="ml-guide.html">MLlib: Main Guide</a></h3>

 <ul>

     <li>
         <a href="ml-statistics.html">

                 Basic statistics

         </a>
     </li>


     <li>
         <a href="ml-datasource.html">

                 Data sources

         </a>
     </li>


     <li>
         <a href="ml-pipeline.html">

                 Pipelines

         </a>
     </li>


     <li>
         <a href="ml-features.html">

                 Extracting, transforming and selecting features

         </a>
     </li>


     <li>
         <a href="ml-classification-regression.html">

                 Classification and Regression

         </a>
     </li>


     <li>
         <a href="ml-clustering.html">

                 Clustering

         </a>
     </li>


     <li>
         <a href="ml-collaborative-filtering.html">

                 Collaborative filtering

         </a>
     </li>


     <li>
         <a href="ml-frequent-pattern-mining.html">

                 Frequent Pattern Mining

         </a>
     </li>


     <li>
         <a href="ml-tuning.html">

                 Model selection and tuning

         </a>
     </li>


     <li>
         <a href="ml-advanced.html">

                 Advanced topics

         </a>
     </li>


 </ul>

         <h3><a href="mllib-guide.html">MLlib: RDD-based API Guide</a></h3>

 <ul>

     <li>
         <a href="mllib-data-types.html">

                 Data types

         </a>
     </li>


     <li>
         <a href="mllib-statistics.html">

                 Basic statistics

         </a>
     </li>


     <li>
         <a href="mllib-classification-regression.html">

                 Classification and regression

         </a>
     </li>


     <li>
         <a href="mllib-collaborative-filtering.html">

                 Collaborative filtering

         </a>
     </li>


     <li>
         <a href="mllib-clustering.html">

                 Clustering

         </a>
     </li>


     <li>
         <a href="mllib-dimensionality-reduction.html">

                 Dimensionality reduction

         </a>
     </li>


     <li>
         <a href="mllib-feature-extraction.html">

                 Feature extraction and transformation

         </a>
     </li>


     <li>
         <a href="mllib-frequent-pattern-mining.html">

                 Frequent pattern mining

         </a>
     </li>


     <li>
         <a href="mllib-evaluation-metrics.html">

                 Evaluation metrics

         </a>
     </li>


     <li>
         <a href="mllib-pmml-model-export.html">

                 PMML model export

         </a>
     </li>


     <li>
         <a href="mllib-optimization.html">

                 Optimization (developer)

         </a>
     </li>


 </ul>

     </div>
 </div>

                 <input id="nav-trigger" class="nav-trigger" checked type="checkbox">
                 <label for="nav-trigger"></label>
                 <div class="content-with-sidebar mr-3" id="content">

                         <h1 class="title">MLlib: RDD-based API</h1>


                     <p>This page documents sections of the MLlib guide for the RDD-based API (the <code class="language-plaintext highlighter-rouge">spark.mllib</code> package).
 Please see the <a href="ml-guide.html">MLlib Main Guide</a> for the DataFrame-based API (the <code class="language-plaintext highlighter-rouge">spark.ml</code> package),
 which is now the primary API for MLlib.</p>

 <ul>
   <li><a href="mllib-data-types.html">Data types</a></li>
   <li><a href="mllib-statistics.html">Basic statistics</a>
     <ul>
       <li><a href="mllib-statistics.html#summary-statistics">summary statistics</a></li>
       <li><a href="mllib-statistics.html#correlations">correlations</a></li>
       <li><a href="mllib-statistics.html#stratified-sampling">stratified sampling</a></li>
       <li><a href="mllib-statistics.html#hypothesis-testing">hypothesis testing</a></li>
       <li><a href="mllib-statistics.html#streaming-significance-testing">streaming significance testing</a></li>
       <li><a href="mllib-statistics.html#random-data-generation">random data generation</a></li>
     </ul>
   </li>
   <li><a href="mllib-classification-regression.html">Classification and regression</a>
     <ul>
       <li><a href="mllib-linear-methods.html">linear models (SVMs, logistic regression, linear regression)</a></li>
       <li><a href="mllib-naive-bayes.html">naive Bayes</a></li>
       <li><a href="mllib-decision-tree.html">decision trees</a></li>
       <li><a href="mllib-ensembles.html">ensembles of trees (Random Forests and Gradient-Boosted Trees)</a></li>
       <li><a href="mllib-isotonic-regression.html">isotonic regression</a></li>
     </ul>
   </li>
   <li><a href="mllib-collaborative-filtering.html">Collaborative filtering</a>
     <ul>
       <li><a href="mllib-collaborative-filtering.html#collaborative-filtering">alternating least squares (ALS)</a></li>
     </ul>
   </li>
   <li><a href="mllib-clustering.html">Clustering</a>
     <ul>
       <li><a href="mllib-clustering.html#k-means">k-means</a></li>
       <li><a href="mllib-clustering.html#gaussian-mixture">Gaussian mixture</a></li>
       <li><a href="mllib-clustering.html#power-iteration-clustering-pic">power iteration clustering (PIC)</a></li>
       <li><a href="mllib-clustering.html#latent-dirichlet-allocation-lda">latent Dirichlet allocation (LDA)</a></li>
       <li><a href="mllib-clustering.html#bisecting-k-means">bisecting k-means</a></li>
       <li><a href="mllib-clustering.html#streaming-k-means">streaming k-means</a></li>
     </ul>
   </li>
   <li><a href="mllib-dimensionality-reduction.html">Dimensionality reduction</a>
     <ul>
       <li><a href="mllib-dimensionality-reduction.html#singular-value-decomposition-svd">singular value decomposition (SVD)</a></li>
       <li><a href="mllib-dimensionality-reduction.html#principal-component-analysis-pca">principal component analysis (PCA)</a></li>
     </ul>
   </li>
   <li><a href="mllib-feature-extraction.html">Feature extraction and transformation</a></li>
   <li><a href="mllib-frequent-pattern-mining.html">Frequent pattern mining</a>
     <ul>
       <li><a href="mllib-frequent-pattern-mining.html#fp-growth">FP-growth</a></li>
       <li><a href="mllib-frequent-pattern-mining.html#association-rules">association rules</a></li>
       <li><a href="mllib-frequent-pattern-mining.html#prefixspan">PrefixSpan</a></li>
     </ul>
   </li>
   <li><a href="mllib-evaluation-metrics.html">Evaluation metrics</a></li>
   <li><a href="mllib-pmml-model-export.html">PMML model export</a></li>
   <li><a href="mllib-optimization.html">Optimization (developer)</a>
     <ul>
       <li><a href="mllib-optimization.html#stochastic-gradient-descent-sgd">stochastic gradient descent</a></li>
       <li><a href="mllib-optimization.html#limited-memory-bfgs-l-bfgs">limited-memory BFGS (L-BFGS)</a></li>
     </ul>
   </li>
 </ul>


                 </div>

              <!-- /container -->
         </div>

         <script src="js/vendor/jquery-3.5.1.min.js"></script>
         <script src="js/vendor/bootstrap.bundle.min.js"></script>

         <script src="js/vendor/anchor.min.js"></script>
         <script src="js/main.js"></script>

         <script type="text/javascript" src="js/vendor/docsearch.min.js"></script>
         <script type="text/javascript">
             // DocSearch is entirely free and automated. DocSearch is built in two parts:
             // 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link
             //    in your website and extract content from every page it traverses. It then pushes this
             //    content to an Algolia index.
             // 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index
             //    to your search input and display its results in a dropdown UI. If you want to find more
             //    details on how works DocSearch, check the docs of DocSearch.
             docsearch({
     apiKey: 'd62f962a82bc9abb53471cb7b89da35e',
     appId: 'RAI69RXRSK',
     indexName: 'apache_spark',
     inputSelector: '#docsearch-input',
     enhancedSearchInput: true,
     algoliaOptions: {
       'facetFilters': ["version:4.1.0-preview1"]
     },
     debug: false // Set debug to true if you want to inspect the dropdown
 });

         </script>

         <!-- MathJax Section -->
         <script type="text/x-mathjax-config">
             MathJax.Hub.Config({
                 TeX: { equationNumbers: { autoNumber: "AMS" } }
             });
         </script>
         <script>
             // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
             // We could use "//cdn.mathjax...", but that won't support "file://".
             (function(d, script) {
                 script = d.createElement('script');
                 script.type = 'text/javascript';
                 script.async = true;
                 script.onload = function(){
                     MathJax.Hub.Config({
                         tex2jax: {
                             inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
                             displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
                             processEscapes: true,
                             skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
                         }
                     });
                 };
                 script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
                     'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
                     '?config=TeX-AMS-MML_HTMLorMML';
                 d.getElementsByTagName('head')[0].appendChild(script);
             }(document));
         </script>
     </body>
 </html>




	<!DOCTYPE html>
	<html class="no-js">
	<head>
	<meta charset="utf-8">
	<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">

	<title>MLlib: RDD-based API - Spark 4.1.0-preview1 Documentation</title>





	<link rel="stylesheet" href="css/bootstrap.min.css">
	<link rel="preconnect" href="https://fonts.googleapis.com">
	<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
	<link href="https://fonts.googleapis.com/css2?family=DM+Sans:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&Courier+Prime:wght@400;700&display=swap" rel="stylesheet">
	<link href="css/custom.css" rel="stylesheet">
	<script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>

	<link rel="stylesheet" href="css/pygments-default.css">
	<link rel="stylesheet" href="css/docsearch.min.css" />
	<link rel="stylesheet" href="css/docsearch.css">


	<!-- Matomo -->
	<script>
	var _paq = window._paq = window._paq \|\| [];
	/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
	_paq.push(["disableCookies"]);
	_paq.push(['trackPageView']);
	_paq.push(['enableLinkTracking']);
	(function() {
	var u="https://analytics.apache.org/";
	_paq.push(['setTrackerUrl', u+'matomo.php']);
	_paq.push(['setSiteId', '40']);
	var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
	g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
	})();
	</script>
	<!-- End Matomo Code -->


	</head>
	<body class="global">
	<!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->
	<nav class="navbar navbar-expand-lg navbar-dark p-0 px-4 fixed-top" style="background: #1d6890;" id="topbar">
	<div class="navbar-brand"><a href="index.html">
	<img src="https://spark.apache.org/images/spark-logo-rev.svg" width="141" height="72"/></a><span class="version">4.1.0-preview1</span>
	</div>
	<button class="navbar-toggler" type="button" data-toggle="collapse"
	data-target="#navbarCollapse" aria-controls="navbarCollapse"
	aria-expanded="false" aria-label="Toggle navigation">
	<span class="navbar-toggler-icon"></span>
	</button>
	<div class="collapse navbar-collapse" id="navbarCollapse">
	<ul class="navbar-nav me-auto">
	<li class="nav-item"><a href="index.html" class="nav-link">Overview</a></li>

	<li class="nav-item dropdown">
	<a href="#" class="nav-link dropdown-toggle" id="navbarQuickStart" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Programming Guides</a>
	<div class="dropdown-menu" aria-labelledby="navbarQuickStart">
	<a class="dropdown-item" href="quick-start.html">Quick Start</a>
	<a class="dropdown-item" href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a>
	<a class="dropdown-item" href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a>
	<a class="dropdown-item" href="streaming/index.html">Structured Streaming</a>
	<a class="dropdown-item" href="streaming-programming-guide.html">Spark Streaming (DStreams)</a>
	<a class="dropdown-item" href="ml-guide.html">MLlib (Machine Learning)</a>
	<a class="dropdown-item" href="graphx-programming-guide.html">GraphX (Graph Processing)</a>
	<a class="dropdown-item" href="sparkr.html">SparkR (R on Spark)</a>
	<a class="dropdown-item" href="api/python/getting_started/index.html">PySpark (Python on Spark)</a>
	<a class="dropdown-item" href="declarative-pipelines-programming-guide.html">Declarative Pipelines</a>
	</div>
	</li>

	<li class="nav-item dropdown">
	<a href="#" class="nav-link dropdown-toggle" id="navbarAPIDocs" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">API Docs</a>
	<div class="dropdown-menu" aria-labelledby="navbarAPIDocs">
	<a class="dropdown-item" href="api/python/index.html">Python</a>
	<a class="dropdown-item" href="api/scala/org/apache/spark/index.html">Scala</a>
	<a class="dropdown-item" href="api/java/index.html">Java</a>
	<a class="dropdown-item" href="api/R/index.html">R</a>
	<a class="dropdown-item" href="api/sql/index.html">SQL, Built-in Functions</a>
	</div>
	</li>

	<li class="nav-item dropdown">
	<a href="#" class="nav-link dropdown-toggle" id="navbarDeploying" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Deploying</a>
	<div class="dropdown-menu" aria-labelledby="navbarDeploying">
	<a class="dropdown-item" href="cluster-overview.html">Overview</a>
	<a class="dropdown-item" href="submitting-applications.html">Submitting Applications</a>
	<div class="dropdown-divider"></div>
	<a class="dropdown-item" href="spark-standalone.html">Spark Standalone</a>
	<a class="dropdown-item" href="running-on-yarn.html">YARN</a>
	<a class="dropdown-item" href="running-on-kubernetes.html">Kubernetes</a>
	</div>
	</li>

	<li class="nav-item dropdown">
	<a href="#" class="nav-link dropdown-toggle" id="navbarMore" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
	<div class="dropdown-menu" aria-labelledby="navbarMore">
	<a class="dropdown-item" href="configuration.html">Configuration</a>
	<a class="dropdown-item" href="monitoring.html">Monitoring</a>
	<a class="dropdown-item" href="tuning.html">Tuning Guide</a>
	<a class="dropdown-item" href="job-scheduling.html">Job Scheduling</a>
	<a class="dropdown-item" href="security.html">Security</a>
	<a class="dropdown-item" href="hardware-provisioning.html">Hardware Provisioning</a>
	<a class="dropdown-item" href="migration-guide.html">Migration Guide</a>
	<div class="dropdown-divider"></div>
	<a class="dropdown-item" href="building-spark.html">Building Spark</a>
	<a class="dropdown-item" href="https://spark.apache.org/contributing.html">Contributing to Spark</a>
	<a class="dropdown-item" href="https://spark.apache.org/third-party-projects.html">Third Party Projects</a>
	</div>
	</li>

	<li class="nav-item">
	<input type="text" id="docsearch-input" placeholder="Search the docs…">
	</li>
	</ul>
	<!--<span class="navbar-text navbar-right"><span class="version-text">v4.1.0-preview1</span></span>-->
	</div>
	</nav>



	<div class="container">


	<div class="left-menu-wrapper">
	<div class="left-menu">
	<h3><a href="ml-guide.html">MLlib: Main Guide</a></h3>

	<ul>

	<li>
	<a href="ml-statistics.html">

	Basic statistics

	</a>
	</li>



	<li>
	<a href="ml-datasource.html">

	Data sources

	</a>
	</li>



	<li>
	<a href="ml-pipeline.html">

	Pipelines

	</a>
	</li>



	<li>
	<a href="ml-features.html">

	Extracting, transforming and selecting features

	</a>
	</li>



	<li>
	<a href="ml-classification-regression.html">

	Classification and Regression

	</a>
	</li>



	<li>
	<a href="ml-clustering.html">

	Clustering

	</a>
	</li>



	<li>
	<a href="ml-collaborative-filtering.html">

	Collaborative filtering

	</a>
	</li>



	<li>
	<a href="ml-frequent-pattern-mining.html">

	Frequent Pattern Mining

	</a>
	</li>



	<li>
	<a href="ml-tuning.html">

	Model selection and tuning

	</a>
	</li>



	<li>
	<a href="ml-advanced.html">

	Advanced topics

	</a>
	</li>



	</ul>

	<h3><a href="mllib-guide.html">MLlib: RDD-based API Guide</a></h3>

	<ul>

	<li>
	<a href="mllib-data-types.html">

	Data types

	</a>
	</li>



	<li>
	<a href="mllib-statistics.html">

	Basic statistics

	</a>
	</li>



	<li>
	<a href="mllib-classification-regression.html">

	Classification and regression

	</a>
	</li>



	<li>
	<a href="mllib-collaborative-filtering.html">

	Collaborative filtering

	</a>
	</li>



	<li>
	<a href="mllib-clustering.html">

	Clustering

	</a>
	</li>



	<li>
	<a href="mllib-dimensionality-reduction.html">

	Dimensionality reduction

	</a>
	</li>



	<li>
	<a href="mllib-feature-extraction.html">

	Feature extraction and transformation

	</a>
	</li>



	<li>
	<a href="mllib-frequent-pattern-mining.html">

	Frequent pattern mining

	</a>
	</li>



	<li>
	<a href="mllib-evaluation-metrics.html">

	Evaluation metrics

	</a>
	</li>



	<li>
	<a href="mllib-pmml-model-export.html">

	PMML model export

	</a>
	</li>



	<li>
	<a href="mllib-optimization.html">

	Optimization (developer)

	</a>
	</li>



	</ul>

	</div>
	</div>

	<input id="nav-trigger" class="nav-trigger" checked type="checkbox">
	<label for="nav-trigger"></label>
	<div class="content-with-sidebar mr-3" id="content">

	<h1 class="title">MLlib: RDD-based API</h1>


	<p>This page documents sections of the MLlib guide for the RDD-based API (the <code class="language-plaintext highlighter-rouge">spark.mllib</code> package).
	Please see the <a href="ml-guide.html">MLlib Main Guide</a> for the DataFrame-based API (the <code class="language-plaintext highlighter-rouge">spark.ml</code> package),
	which is now the primary API for MLlib.</p>

	<ul>
	<li><a href="mllib-data-types.html">Data types</a></li>
	<li><a href="mllib-statistics.html">Basic statistics</a>
	<ul>
	<li><a href="mllib-statistics.html#summary-statistics">summary statistics</a></li>
	<li><a href="mllib-statistics.html#correlations">correlations</a></li>
	<li><a href="mllib-statistics.html#stratified-sampling">stratified sampling</a></li>
	<li><a href="mllib-statistics.html#hypothesis-testing">hypothesis testing</a></li>
	<li><a href="mllib-statistics.html#streaming-significance-testing">streaming significance testing</a></li>
	<li><a href="mllib-statistics.html#random-data-generation">random data generation</a></li>
	</ul>
	</li>
	<li><a href="mllib-classification-regression.html">Classification and regression</a>
	<ul>
	<li><a href="mllib-linear-methods.html">linear models (SVMs, logistic regression, linear regression)</a></li>
	<li><a href="mllib-naive-bayes.html">naive Bayes</a></li>
	<li><a href="mllib-decision-tree.html">decision trees</a></li>
	<li><a href="mllib-ensembles.html">ensembles of trees (Random Forests and Gradient-Boosted Trees)</a></li>
	<li><a href="mllib-isotonic-regression.html">isotonic regression</a></li>
	</ul>
	</li>
	<li><a href="mllib-collaborative-filtering.html">Collaborative filtering</a>
	<ul>
	<li><a href="mllib-collaborative-filtering.html#collaborative-filtering">alternating least squares (ALS)</a></li>
	</ul>
	</li>
	<li><a href="mllib-clustering.html">Clustering</a>
	<ul>
	<li><a href="mllib-clustering.html#k-means">k-means</a></li>
	<li><a href="mllib-clustering.html#gaussian-mixture">Gaussian mixture</a></li>
	<li><a href="mllib-clustering.html#power-iteration-clustering-pic">power iteration clustering (PIC)</a></li>
	<li><a href="mllib-clustering.html#latent-dirichlet-allocation-lda">latent Dirichlet allocation (LDA)</a></li>
	<li><a href="mllib-clustering.html#bisecting-k-means">bisecting k-means</a></li>
	<li><a href="mllib-clustering.html#streaming-k-means">streaming k-means</a></li>
	</ul>
	</li>
	<li><a href="mllib-dimensionality-reduction.html">Dimensionality reduction</a>
	<ul>
	<li><a href="mllib-dimensionality-reduction.html#singular-value-decomposition-svd">singular value decomposition (SVD)</a></li>
	<li><a href="mllib-dimensionality-reduction.html#principal-component-analysis-pca">principal component analysis (PCA)</a></li>
	</ul>
	</li>
	<li><a href="mllib-feature-extraction.html">Feature extraction and transformation</a></li>
	<li><a href="mllib-frequent-pattern-mining.html">Frequent pattern mining</a>
	<ul>
	<li><a href="mllib-frequent-pattern-mining.html#fp-growth">FP-growth</a></li>
	<li><a href="mllib-frequent-pattern-mining.html#association-rules">association rules</a></li>
	<li><a href="mllib-frequent-pattern-mining.html#prefixspan">PrefixSpan</a></li>
	</ul>
	</li>
	<li><a href="mllib-evaluation-metrics.html">Evaluation metrics</a></li>
	<li><a href="mllib-pmml-model-export.html">PMML model export</a></li>
	<li><a href="mllib-optimization.html">Optimization (developer)</a>
	<ul>
	<li><a href="mllib-optimization.html#stochastic-gradient-descent-sgd">stochastic gradient descent</a></li>
	<li><a href="mllib-optimization.html#limited-memory-bfgs-l-bfgs">limited-memory BFGS (L-BFGS)</a></li>
	</ul>
	</li>
	</ul>


	</div>

	<!-- /container -->
	</div>

	<script src="js/vendor/jquery-3.5.1.min.js"></script>
	<script src="js/vendor/bootstrap.bundle.min.js"></script>

	<script src="js/vendor/anchor.min.js"></script>
	<script src="js/main.js"></script>

	<script type="text/javascript" src="js/vendor/docsearch.min.js"></script>
	<script type="text/javascript">
	// DocSearch is entirely free and automated. DocSearch is built in two parts:
	// 1. a crawler which we run on our own infrastructure every 24 hours. It follows every link
	// in your website and extract content from every page it traverses. It then pushes this
	// content to an Algolia index.
	// 2. a JavaScript snippet to be inserted in your website that will bind this Algolia index
	// to your search input and display its results in a dropdown UI. If you want to find more
	// details on how works DocSearch, check the docs of DocSearch.
	docsearch({
	apiKey: 'd62f962a82bc9abb53471cb7b89da35e',
	appId: 'RAI69RXRSK',
	indexName: 'apache_spark',
	inputSelector: '#docsearch-input',
	enhancedSearchInput: true,
	algoliaOptions: {
	'facetFilters': ["version:4.1.0-preview1"]
	},
	debug: false // Set debug to true if you want to inspect the dropdown
	});

	</script>

	<!-- MathJax Section -->
	<script type="text/x-mathjax-config">
	MathJax.Hub.Config({
	TeX: { equationNumbers: { autoNumber: "AMS" } }
	});
	</script>
	<script>
	// Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
	// We could use "//cdn.mathjax...", but that won't support "file://".
	(function(d, script) {
	script = d.createElement('script');
	script.type = 'text/javascript';
	script.async = true;
	script.onload = function(){
	MathJax.Hub.Config({
	tex2jax: {
	inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
	displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
	processEscapes: true,
	skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
	}
	});
	};
	script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
	'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
	'?config=TeX-AMS-MML_HTMLorMML';
	d.getElementsByTagName('head')[0].appendChild(script);
	}(document));
	</script>
	</body>
	</html>