site/docs/2.2.1/mllib-dimensionality-reduction.html - spark-website - Git at Google


 <!DOCTYPE html>
 <!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
 <!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
 <!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
     <head>
         <meta charset="utf-8">
         <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
         <title>Dimensionality Reduction - RDD-based API - Spark 2.2.1 Documentation</title>


         <link rel="stylesheet" href="css/bootstrap.min.css">
         <style>
             body {
                 padding-top: 60px;
                 padding-bottom: 40px;
             }
         </style>
         <meta name="viewport" content="width=device-width">
         <link rel="stylesheet" href="css/bootstrap-responsive.min.css">
         <link rel="stylesheet" href="css/main.css">

         <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>

         <link rel="stylesheet" href="css/pygments-default.css">


         <!-- Google analytics script -->
         <script type="text/javascript">
           var _gaq = _gaq || [];
           _gaq.push(['_setAccount', 'UA-32518208-2']);
           _gaq.push(['_trackPageview']);

           (function() {
             var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
             ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
             var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
           })();
         </script>


     </head>
     <body>
         <!--[if lt IE 7]>
             <p class="chromeframe">You are using an outdated browser. <a href="http://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p>
         <![endif]-->

         <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->

         <div class="navbar navbar-fixed-top" id="topbar">
             <div class="navbar-inner">
                 <div class="container">
                     <div class="brand"><a href="index.html">
                       <img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">2.2.1</span>
                     </div>
                     <ul class="nav">
                         <!--TODO(andyk): Add class="active" attribute to li some how.-->
                         <li><a href="index.html">Overview</a></li>

                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">Programming Guides<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="quick-start.html">Quick Start</a></li>
                                 <li><a href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a></li>
                                 <li><a href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a></li>
                                 <li><a href="structured-streaming-programming-guide.html">Structured Streaming</a></li>
                                 <li><a href="streaming-programming-guide.html">Spark Streaming (DStreams)</a></li>
                                 <li><a href="ml-guide.html">MLlib (Machine Learning)</a></li>
                                 <li><a href="graphx-programming-guide.html">GraphX (Graph Processing)</a></li>
                                 <li><a href="sparkr.html">SparkR (R on Spark)</a></li>
                             </ul>
                         </li>

                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="api/scala/index.html#org.apache.spark.package">Scala</a></li>
                                 <li><a href="api/java/index.html">Java</a></li>
                                 <li><a href="api/python/index.html">Python</a></li>
                                 <li><a href="api/R/index.html">R</a></li>
                             </ul>
                         </li>

                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">Deploying<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="cluster-overview.html">Overview</a></li>
                                 <li><a href="submitting-applications.html">Submitting Applications</a></li>
                                 <li class="divider"></li>
                                 <li><a href="spark-standalone.html">Spark Standalone</a></li>
                                 <li><a href="running-on-mesos.html">Mesos</a></li>
                                 <li><a href="running-on-yarn.html">YARN</a></li>
                             </ul>
                         </li>

                         <li class="dropdown">
                             <a href="api.html" class="dropdown-toggle" data-toggle="dropdown">More<b class="caret"></b></a>
                             <ul class="dropdown-menu">
                                 <li><a href="configuration.html">Configuration</a></li>
                                 <li><a href="monitoring.html">Monitoring</a></li>
                                 <li><a href="tuning.html">Tuning Guide</a></li>
                                 <li><a href="job-scheduling.html">Job Scheduling</a></li>
                                 <li><a href="security.html">Security</a></li>
                                 <li><a href="hardware-provisioning.html">Hardware Provisioning</a></li>
                                 <li class="divider"></li>
                                 <li><a href="building-spark.html">Building Spark</a></li>
                                 <li><a href="http://spark.apache.org/contributing.html">Contributing to Spark</a></li>
                                 <li><a href="http://spark.apache.org/third-party-projects.html">Third Party Projects</a></li>
                             </ul>
                         </li>
                     </ul>
                     <!--<p class="navbar-text pull-right"><span class="version-text">v2.2.1</span></p>-->
                 </div>
             </div>
         </div>

         <div class="container-wrapper">


                 <div class="left-menu-wrapper">
     <div class="left-menu">
         <h3><a href="ml-guide.html">MLlib: Main Guide</a></h3>

 <ul>

     <li>
         <a href="ml-statistics.html">

                 Basic statistics

         </a>
     </li>


     <li>
         <a href="ml-pipeline.html">

                 Pipelines

         </a>
     </li>


     <li>
         <a href="ml-features.html">

                 Extracting, transforming and selecting features

         </a>
     </li>


     <li>
         <a href="ml-classification-regression.html">

                 Classification and Regression

         </a>
     </li>


     <li>
         <a href="ml-clustering.html">

                 Clustering

         </a>
     </li>


     <li>
         <a href="ml-collaborative-filtering.html">

                 Collaborative filtering

         </a>
     </li>


     <li>
         <a href="ml-frequent-pattern-mining.html">

                 Frequent Pattern Mining

         </a>
     </li>


     <li>
         <a href="ml-tuning.html">

                 Model selection and tuning

         </a>
     </li>


     <li>
         <a href="ml-advanced.html">

                 Advanced topics

         </a>
     </li>


 </ul>

         <h3><a href="mllib-guide.html">MLlib: RDD-based API Guide</a></h3>

 <ul>

     <li>
         <a href="mllib-data-types.html">

                 Data types

         </a>
     </li>


     <li>
         <a href="mllib-statistics.html">

                 Basic statistics

         </a>
     </li>


     <li>
         <a href="mllib-classification-regression.html">

                 Classification and regression

         </a>
     </li>


     <li>
         <a href="mllib-collaborative-filtering.html">

                 Collaborative filtering

         </a>
     </li>


     <li>
         <a href="mllib-clustering.html">

                 Clustering

         </a>
     </li>


     <li>
         <a href="mllib-dimensionality-reduction.html">

                 <b>Dimensionality reduction</b>

         </a>
     </li>


 <ul>

     <li>
         <a href="mllib-dimensionality-reduction.html#singular-value-decomposition-svd">

                 singular value decomposition (SVD)

         </a>
     </li>


     <li>
         <a href="mllib-dimensionality-reduction.html#principal-component-analysis-pca">

                 principal component analysis (PCA)

         </a>
     </li>


 </ul>


     <li>
         <a href="mllib-feature-extraction.html">

                 Feature extraction and transformation

         </a>
     </li>


     <li>
         <a href="mllib-frequent-pattern-mining.html">

                 Frequent pattern mining

         </a>
     </li>


     <li>
         <a href="mllib-evaluation-metrics.html">

                 Evaluation metrics

         </a>
     </li>


     <li>
         <a href="mllib-pmml-model-export.html">

                 PMML model export

         </a>
     </li>


     <li>
         <a href="mllib-optimization.html">

                 Optimization (developer)

         </a>
     </li>


 </ul>

     </div>
 </div>
                 <input id="nav-trigger" class="nav-trigger" checked type="checkbox">
                 <label for="nav-trigger"></label>
                 <div class="content-with-sidebar" id="content">

                         <h1 class="title">Dimensionality Reduction - RDD-based API</h1>


                     <ul id="markdown-toc">
   <li><a href="#singular-value-decomposition-svd" id="markdown-toc-singular-value-decomposition-svd">Singular value decomposition (SVD)</a>    <ul>
       <li><a href="#performance" id="markdown-toc-performance">Performance</a></li>
       <li><a href="#svd-example" id="markdown-toc-svd-example">SVD Example</a></li>
     </ul>
   </li>
   <li><a href="#principal-component-analysis-pca" id="markdown-toc-principal-component-analysis-pca">Principal component analysis (PCA)</a></li>
 </ul>

 <p><a href="http://en.wikipedia.org/wiki/Dimensionality_reduction">Dimensionality reduction</a> is the process
 of reducing the number of variables under consideration.
 It can be used to extract latent features from raw and noisy features
 or compress data while maintaining the structure.
 <code>spark.mllib</code> provides support for dimensionality reduction on the <a href="mllib-data-types.html#rowmatrix">RowMatrix</a> class.</p>

 <h2 id="singular-value-decomposition-svd">Singular value decomposition (SVD)</h2>

 <p><a href="http://en.wikipedia.org/wiki/Singular_value_decomposition">Singular value decomposition (SVD)</a>
 factorizes a matrix into three matrices: $U$, $\Sigma$, and $V$ such that</p>

 <p><code>\[
 A = U \Sigma V^T,
 \]</code></p>

 <p>where</p>

 <ul>
   <li>$U$ is an orthonormal matrix, whose columns are called left singular vectors,</li>
   <li>$\Sigma$ is a diagonal matrix with non-negative diagonals in descending order,
 whose diagonals are called singular values,</li>
   <li>$V$ is an orthonormal matrix, whose columns are called right singular vectors.</li>
 </ul>

 <p>For large matrices, usually we don&#8217;t need the complete factorization but only the top singular
 values and its associated singular vectors.  This can save storage, de-noise
 and recover the low-rank structure of the matrix.</p>

 <p>If we keep the top $k$ singular values, then the dimensions of the resulting low-rank matrix will be:</p>

 <ul>
   <li><code>$U$</code>: <code>$m \times k$</code>,</li>
   <li><code>$\Sigma$</code>: <code>$k \times k$</code>,</li>
   <li><code>$V$</code>: <code>$n \times k$</code>.</li>
 </ul>

 <h3 id="performance">Performance</h3>
 <p>We assume $n$ is smaller than $m$. The singular values and the right singular vectors are derived
 from the eigenvalues and the eigenvectors of the Gramian matrix $A^T A$. The matrix
 storing the left singular vectors $U$, is computed via matrix multiplication as
 $U = A (V S^{-1})$, if requested by the user via the computeU parameter.
 The actual method to use is determined automatically based on the computational cost:</p>

 <ul>
   <li>If $n$ is small ($n &lt; 100$) or $k$ is large compared with $n$ ($k &gt; n / 2$), we compute the Gramian matrix
 first and then compute its top eigenvalues and eigenvectors locally on the driver.
 This requires a single pass with $O(n^2)$ storage on each executor and on the driver, and
 $O(n^2 k)$ time on the driver.</li>
   <li>Otherwise, we compute $(A^T A) v$ in a distributive way and send it to
 <a href="http://www.caam.rice.edu/software/ARPACK/">ARPACK</a> to
 compute $(A^T A)$&#8217;s top eigenvalues and eigenvectors on the driver node. This requires $O(k)$
 passes, $O(n)$ storage on each executor, and $O(n k)$ storage on the driver.</li>
 </ul>

 <h3 id="svd-example">SVD Example</h3>

 <p><code>spark.mllib</code> provides SVD functionality to row-oriented matrices, provided in the
 <a href="mllib-data-types.html#rowmatrix">RowMatrix</a> class.</p>

 <div class="codetabs">
 <div data-lang="scala">
     <p>Refer to the <a href="api/scala/index.html#org.apache.spark.mllib.linalg.SingularValueDecomposition"><code>SingularValueDecomposition</code> Scala docs</a> for details on the API.</p>

     <div class="highlight"><pre><span></span><span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.Matrix</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.SingularValueDecomposition</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vector</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vectors</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.distributed.RowMatrix</span>

 <span class="k">val</span> <span class="n">data</span> <span class="k">=</span> <span class="nc">Array</span><span class="o">(</span>
   <span class="nc">Vectors</span><span class="o">.</span><span class="n">sparse</span><span class="o">(</span><span class="mi">5</span><span class="o">,</span> <span class="nc">Seq</span><span class="o">((</span><span class="mi">1</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span> <span class="o">(</span><span class="mi">3</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">))),</span>
   <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mf">2.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">3.0</span><span class="o">,</span> <span class="mf">4.0</span><span class="o">,</span> <span class="mf">5.0</span><span class="o">),</span>
   <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mf">4.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">6.0</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">))</span>

 <span class="k">val</span> <span class="n">rows</span> <span class="k">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="o">(</span><span class="n">data</span><span class="o">)</span>

 <span class="k">val</span> <span class="n">mat</span><span class="k">:</span> <span class="kt">RowMatrix</span> <span class="o">=</span> <span class="k">new</span> <span class="nc">RowMatrix</span><span class="o">(</span><span class="n">rows</span><span class="o">)</span>

 <span class="c1">// Compute the top 5 singular values and corresponding singular vectors.</span>
 <span class="k">val</span> <span class="n">svd</span><span class="k">:</span> <span class="kt">SingularValueDecomposition</span><span class="o">[</span><span class="kt">RowMatrix</span>, <span class="kt">Matrix</span><span class="o">]</span> <span class="k">=</span> <span class="n">mat</span><span class="o">.</span><span class="n">computeSVD</span><span class="o">(</span><span class="mi">5</span><span class="o">,</span> <span class="n">computeU</span> <span class="k">=</span> <span class="kc">true</span><span class="o">)</span>
 <span class="k">val</span> <span class="n">U</span><span class="k">:</span> <span class="kt">RowMatrix</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="n">U</span>  <span class="c1">// The U factor is a RowMatrix.</span>
 <span class="k">val</span> <span class="n">s</span><span class="k">:</span> <span class="kt">Vector</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="n">s</span>     <span class="c1">// The singular values are stored in a local dense vector.</span>
 <span class="k">val</span> <span class="n">V</span><span class="k">:</span> <span class="kt">Matrix</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="n">V</span>     <span class="c1">// The V factor is a local dense matrix.</span>
 </pre></div>
     <div><small>Find full example code at "examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala" in the Spark repo.</small></div>

     <p>The same code applies to <code>IndexedRowMatrix</code> if <code>U</code> is defined as an
 <code>IndexedRowMatrix</code>.</p>
   </div>
 <div data-lang="java">
     <p>Refer to the <a href="api/java/org/apache/spark/mllib/linalg/SingularValueDecomposition.html"><code>SingularValueDecomposition</code> Java docs</a> for details on the API.</p>

     <div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">java.util.Arrays</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">java.util.List</span><span class="o">;</span>

 <span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaRDD</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaSparkContext</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.Matrix</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.SingularValueDecomposition</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vector</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vectors</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.distributed.RowMatrix</span><span class="o">;</span>

 <span class="n">List</span><span class="o">&lt;</span><span class="n">Vector</span><span class="o">&gt;</span> <span class="n">data</span> <span class="o">=</span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span>
         <span class="n">Vectors</span><span class="o">.</span><span class="na">sparse</span><span class="o">(</span><span class="mi">5</span><span class="o">,</span> <span class="k">new</span> <span class="kt">int</span><span class="o">[]</span> <span class="o">{</span><span class="mi">1</span><span class="o">,</span> <span class="mi">3</span><span class="o">},</span> <span class="k">new</span> <span class="kt">double</span><span class="o">[]</span> <span class="o">{</span><span class="mf">1.0</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">}),</span>
         <span class="n">Vectors</span><span class="o">.</span><span class="na">dense</span><span class="o">(</span><span class="mf">2.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">3.0</span><span class="o">,</span> <span class="mf">4.0</span><span class="o">,</span> <span class="mf">5.0</span><span class="o">),</span>
         <span class="n">Vectors</span><span class="o">.</span><span class="na">dense</span><span class="o">(</span><span class="mf">4.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">6.0</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">)</span>
 <span class="o">);</span>

 <span class="n">JavaRDD</span><span class="o">&lt;</span><span class="n">Vector</span><span class="o">&gt;</span> <span class="n">rows</span> <span class="o">=</span> <span class="n">jsc</span><span class="o">.</span><span class="na">parallelize</span><span class="o">(</span><span class="n">data</span><span class="o">);</span>

 <span class="c1">// Create a RowMatrix from JavaRDD&lt;Vector&gt;.</span>
 <span class="n">RowMatrix</span> <span class="n">mat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">RowMatrix</span><span class="o">(</span><span class="n">rows</span><span class="o">.</span><span class="na">rdd</span><span class="o">());</span>

 <span class="c1">// Compute the top 5 singular values and corresponding singular vectors.</span>
 <span class="n">SingularValueDecomposition</span><span class="o">&lt;</span><span class="n">RowMatrix</span><span class="o">,</span> <span class="n">Matrix</span><span class="o">&gt;</span> <span class="n">svd</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="na">computeSVD</span><span class="o">(</span><span class="mi">5</span><span class="o">,</span> <span class="kc">true</span><span class="o">,</span> <span class="mf">1.0E-9d</span><span class="o">);</span>
 <span class="n">RowMatrix</span> <span class="n">U</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="na">U</span><span class="o">();</span>  <span class="c1">// The U factor is a RowMatrix.</span>
 <span class="n">Vector</span> <span class="n">s</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="na">s</span><span class="o">();</span>     <span class="c1">// The singular values are stored in a local dense vector.</span>
 <span class="n">Matrix</span> <span class="n">V</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="na">V</span><span class="o">();</span>     <span class="c1">// The V factor is a local dense matrix.</span>
 </pre></div>
     <div><small>Find full example code at "examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java" in the Spark repo.</small></div>

     <p>The same code applies to <code>IndexedRowMatrix</code> if <code>U</code> is defined as an
 <code>IndexedRowMatrix</code>.</p>
   </div>
 <div data-lang="python">
     <p>Refer to the <a href="api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.SingularValueDecomposition"><code>SingularValueDecomposition</code> Python docs</a> for details on the API.</p>

     <div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">pyspark.mllib.linalg</span> <span class="kn">import</span> <span class="n">Vectors</span>
 <span class="kn">from</span> <span class="nn">pyspark.mllib.linalg.distributed</span> <span class="kn">import</span> <span class="n">RowMatrix</span>

 <span class="n">rows</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span>
     <span class="n">Vectors</span><span class="o">.</span><span class="n">sparse</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="p">{</span><span class="mi">1</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mi">3</span><span class="p">:</span> <span class="mf">7.0</span><span class="p">}),</span>
     <span class="n">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="p">(</span><span class="mf">2.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">,</span> <span class="mf">4.0</span><span class="p">,</span> <span class="mf">5.0</span><span class="p">),</span>
     <span class="n">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="p">(</span><span class="mf">4.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">,</span> <span class="mf">7.0</span><span class="p">)</span>
 <span class="p">])</span>

 <span class="n">mat</span> <span class="o">=</span> <span class="n">RowMatrix</span><span class="p">(</span><span class="n">rows</span><span class="p">)</span>

 <span class="c1"># Compute the top 5 singular values and corresponding singular vectors.</span>
 <span class="n">svd</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="n">computeSVD</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">computeU</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
 <span class="n">U</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="n">U</span>       <span class="c1"># The U factor is a RowMatrix.</span>
 <span class="n">s</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="n">s</span>       <span class="c1"># The singular values are stored in a local dense vector.</span>
 <span class="n">V</span> <span class="o">=</span> <span class="n">svd</span><span class="o">.</span><span class="n">V</span>       <span class="c1"># The V factor is a local dense matrix.</span>
 </pre></div>
     <div><small>Find full example code at "examples/src/main/python/mllib/svd_example.py" in the Spark repo.</small></div>

     <p>The same code applies to <code>IndexedRowMatrix</code> if <code>U</code> is defined as an
 <code>IndexedRowMatrix</code>.</p>
   </div>
 </div>

 <h2 id="principal-component-analysis-pca">Principal component analysis (PCA)</h2>

 <p><a href="http://en.wikipedia.org/wiki/Principal_component_analysis">Principal component analysis (PCA)</a> is a
 statistical method to find a rotation such that the first coordinate has the largest variance
 possible, and each succeeding coordinate in turn has the largest variance possible. The columns of
 the rotation matrix are called principal components. PCA is used widely in dimensionality reduction.</p>

 <p><code>spark.mllib</code> supports PCA for tall-and-skinny matrices stored in row-oriented format and any Vectors.</p>

 <div class="codetabs">
 <div data-lang="scala">

     <p>The following code demonstrates how to compute principal components on a <code>RowMatrix</code>
 and use them to project the vectors into a low-dimensional space.</p>

     <p>Refer to the <a href="api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix"><code>RowMatrix</code> Scala docs</a> for details on the API.</p>

     <div class="highlight"><pre><span></span><span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.Matrix</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vectors</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.distributed.RowMatrix</span>

 <span class="k">val</span> <span class="n">data</span> <span class="k">=</span> <span class="nc">Array</span><span class="o">(</span>
   <span class="nc">Vectors</span><span class="o">.</span><span class="n">sparse</span><span class="o">(</span><span class="mi">5</span><span class="o">,</span> <span class="nc">Seq</span><span class="o">((</span><span class="mi">1</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span> <span class="o">(</span><span class="mi">3</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">))),</span>
   <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mf">2.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">3.0</span><span class="o">,</span> <span class="mf">4.0</span><span class="o">,</span> <span class="mf">5.0</span><span class="o">),</span>
   <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mf">4.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">6.0</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">))</span>

 <span class="k">val</span> <span class="n">rows</span> <span class="k">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="o">(</span><span class="n">data</span><span class="o">)</span>

 <span class="k">val</span> <span class="n">mat</span><span class="k">:</span> <span class="kt">RowMatrix</span> <span class="o">=</span> <span class="k">new</span> <span class="nc">RowMatrix</span><span class="o">(</span><span class="n">rows</span><span class="o">)</span>

 <span class="c1">// Compute the top 4 principal components.</span>
 <span class="c1">// Principal components are stored in a local dense matrix.</span>
 <span class="k">val</span> <span class="n">pc</span><span class="k">:</span> <span class="kt">Matrix</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="n">computePrincipalComponents</span><span class="o">(</span><span class="mi">4</span><span class="o">)</span>

 <span class="c1">// Project the rows to the linear space spanned by the top 4 principal components.</span>
 <span class="k">val</span> <span class="n">projected</span><span class="k">:</span> <span class="kt">RowMatrix</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="n">multiply</span><span class="o">(</span><span class="n">pc</span><span class="o">)</span>
 </pre></div>
     <div><small>Find full example code at "examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala" in the Spark repo.</small></div>

     <p>The following code demonstrates how to compute principal components on source vectors
 and use them to project the vectors into a low-dimensional space while keeping associated labels:</p>

     <p>Refer to the <a href="api/scala/index.html#org.apache.spark.mllib.feature.PCA"><code>PCA</code> Scala docs</a> for details on the API.</p>

     <div class="highlight"><pre><span></span><span class="k">import</span> <span class="nn">org.apache.spark.mllib.feature.PCA</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vectors</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.mllib.regression.LabeledPoint</span>
 <span class="k">import</span> <span class="nn">org.apache.spark.rdd.RDD</span>

 <span class="k">val</span> <span class="n">data</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">LabeledPoint</span><span class="o">]</span> <span class="k">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="o">(</span><span class="nc">Seq</span><span class="o">(</span>
   <span class="k">new</span> <span class="nc">LabeledPoint</span><span class="o">(</span><span class="mi">0</span><span class="o">,</span> <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">1</span><span class="o">)),</span>
   <span class="k">new</span> <span class="nc">LabeledPoint</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">1</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">1</span><span class="o">,</span> <span class="mi">0</span><span class="o">)),</span>
   <span class="k">new</span> <span class="nc">LabeledPoint</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">1</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">)),</span>
   <span class="k">new</span> <span class="nc">LabeledPoint</span><span class="o">(</span><span class="mi">0</span><span class="o">,</span> <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">)),</span>
   <span class="k">new</span> <span class="nc">LabeledPoint</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="nc">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">1</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">,</span> <span class="mi">0</span><span class="o">))))</span>

 <span class="c1">// Compute the top 5 principal components.</span>
 <span class="k">val</span> <span class="n">pca</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">PCA</span><span class="o">(</span><span class="mi">5</span><span class="o">).</span><span class="n">fit</span><span class="o">(</span><span class="n">data</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">features</span><span class="o">))</span>

 <span class="c1">// Project vectors to the linear space spanned by the top 5 principal</span>
 <span class="c1">// components, keeping the label</span>
 <span class="k">val</span> <span class="n">projected</span> <span class="k">=</span> <span class="n">data</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="n">p</span> <span class="k">=&gt;</span> <span class="n">p</span><span class="o">.</span><span class="n">copy</span><span class="o">(</span><span class="n">features</span> <span class="k">=</span> <span class="n">pca</span><span class="o">.</span><span class="n">transform</span><span class="o">(</span><span class="n">p</span><span class="o">.</span><span class="n">features</span><span class="o">)))</span>
 </pre></div>
     <div><small>Find full example code at "examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala" in the Spark repo.</small></div>

   </div>

 <div data-lang="java">

     <p>The following code demonstrates how to compute principal components on a <code>RowMatrix</code>
 and use them to project the vectors into a low-dimensional space.</p>

     <p>Refer to the <a href="api/java/org/apache/spark/mllib/linalg/distributed/RowMatrix.html"><code>RowMatrix</code> Java docs</a> for details on the API.</p>

     <div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">java.util.Arrays</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">java.util.List</span><span class="o">;</span>

 <span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaRDD</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaSparkContext</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.Matrix</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vector</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vectors</span><span class="o">;</span>
 <span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.distributed.RowMatrix</span><span class="o">;</span>

 <span class="n">List</span><span class="o">&lt;</span><span class="n">Vector</span><span class="o">&gt;</span> <span class="n">data</span> <span class="o">=</span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span>
         <span class="n">Vectors</span><span class="o">.</span><span class="na">sparse</span><span class="o">(</span><span class="mi">5</span><span class="o">,</span> <span class="k">new</span> <span class="kt">int</span><span class="o">[]</span> <span class="o">{</span><span class="mi">1</span><span class="o">,</span> <span class="mi">3</span><span class="o">},</span> <span class="k">new</span> <span class="kt">double</span><span class="o">[]</span> <span class="o">{</span><span class="mf">1.0</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">}),</span>
         <span class="n">Vectors</span><span class="o">.</span><span class="na">dense</span><span class="o">(</span><span class="mf">2.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">3.0</span><span class="o">,</span> <span class="mf">4.0</span><span class="o">,</span> <span class="mf">5.0</span><span class="o">),</span>
         <span class="n">Vectors</span><span class="o">.</span><span class="na">dense</span><span class="o">(</span><span class="mf">4.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">,</span> <span class="mf">6.0</span><span class="o">,</span> <span class="mf">7.0</span><span class="o">)</span>
 <span class="o">);</span>

 <span class="n">JavaRDD</span><span class="o">&lt;</span><span class="n">Vector</span><span class="o">&gt;</span> <span class="n">rows</span> <span class="o">=</span> <span class="n">jsc</span><span class="o">.</span><span class="na">parallelize</span><span class="o">(</span><span class="n">data</span><span class="o">);</span>

 <span class="c1">// Create a RowMatrix from JavaRDD&lt;Vector&gt;.</span>
 <span class="n">RowMatrix</span> <span class="n">mat</span> <span class="o">=</span> <span class="k">new</span> <span class="n">RowMatrix</span><span class="o">(</span><span class="n">rows</span><span class="o">.</span><span class="na">rdd</span><span class="o">());</span>

 <span class="c1">// Compute the top 4 principal components.</span>
 <span class="c1">// Principal components are stored in a local dense matrix.</span>
 <span class="n">Matrix</span> <span class="n">pc</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="na">computePrincipalComponents</span><span class="o">(</span><span class="mi">4</span><span class="o">);</span>

 <span class="c1">// Project the rows to the linear space spanned by the top 4 principal components.</span>
 <span class="n">RowMatrix</span> <span class="n">projected</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="na">multiply</span><span class="o">(</span><span class="n">pc</span><span class="o">);</span>
 </pre></div>
     <div><small>Find full example code at "examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java" in the Spark repo.</small></div>

   </div>

 <div data-lang="python">

     <p>The following code demonstrates how to compute principal components on a <code>RowMatrix</code>
 and use them to project the vectors into a low-dimensional space.</p>

     <p>Refer to the <a href="api/python/pyspark.mllib.html#pyspark.mllib.linalg.distributed.RowMatrix"><code>RowMatrix</code> Python docs</a> for details on the API.</p>

     <div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">pyspark.mllib.linalg</span> <span class="kn">import</span> <span class="n">Vectors</span>
 <span class="kn">from</span> <span class="nn">pyspark.mllib.linalg.distributed</span> <span class="kn">import</span> <span class="n">RowMatrix</span>

 <span class="n">rows</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span>
     <span class="n">Vectors</span><span class="o">.</span><span class="n">sparse</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="p">{</span><span class="mi">1</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mi">3</span><span class="p">:</span> <span class="mf">7.0</span><span class="p">}),</span>
     <span class="n">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="p">(</span><span class="mf">2.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">,</span> <span class="mf">4.0</span><span class="p">,</span> <span class="mf">5.0</span><span class="p">),</span>
     <span class="n">Vectors</span><span class="o">.</span><span class="n">dense</span><span class="p">(</span><span class="mf">4.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mf">6.0</span><span class="p">,</span> <span class="mf">7.0</span><span class="p">)</span>
 <span class="p">])</span>

 <span class="n">mat</span> <span class="o">=</span> <span class="n">RowMatrix</span><span class="p">(</span><span class="n">rows</span><span class="p">)</span>
 <span class="c1"># Compute the top 4 principal components.</span>
 <span class="c1"># Principal components are stored in a local dense matrix.</span>
 <span class="n">pc</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="n">computePrincipalComponents</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>

 <span class="c1"># Project the rows to the linear space spanned by the top 4 principal components.</span>
 <span class="n">projected</span> <span class="o">=</span> <span class="n">mat</span><span class="o">.</span><span class="n">multiply</span><span class="p">(</span><span class="n">pc</span><span class="p">)</span>
 </pre></div>
     <div><small>Find full example code at "examples/src/main/python/mllib/pca_rowmatrix_example.py" in the Spark repo.</small></div>

   </div>
 </div>


                 </div>

              <!-- /container -->
         </div>

         <script src="js/vendor/jquery-1.8.0.min.js"></script>
         <script src="js/vendor/bootstrap.min.js"></script>
         <script src="js/vendor/anchor.min.js"></script>
         <script src="js/main.js"></script>

         <!-- MathJax Section -->
         <script type="text/x-mathjax-config">
             MathJax.Hub.Config({
                 TeX: { equationNumbers: { autoNumber: "AMS" } }
             });
         </script>
         <script>
             // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
             // We could use "//cdn.mathjax...", but that won't support "file://".
             (function(d, script) {
                 script = d.createElement('script');
                 script.type = 'text/javascript';
                 script.async = true;
                 script.onload = function(){
                     MathJax.Hub.Config({
                         tex2jax: {
                             inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
                             displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
                             processEscapes: true,
                             skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
                         }
                     });
                 };
                 script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
                     'cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML';
                 d.getElementsByTagName('head')[0].appendChild(script);
             }(document));
         </script>
     </body>
 </html>