site/docs/4.0.0-preview1/api/R/articles/sparkr-vignettes.html - spark-website - Git at Google

 <!DOCTYPE html>
 <!-- Generated by pkgdown: do not edit by hand --><html lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
 <meta charset="utf-8">
 <meta http-equiv="X-UA-Compatible" content="IE=edge">
 <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 <meta name="description" content="SparkR">
 <!-- Inform modern browsers that this page supports both dark and light color schemes,
   and the page author prefers light. --><meta name="color-scheme" content="dark light">
 <script>
   // If `prefers-color-scheme` is not supported, fall back to light mode.
   // i.e. In this case, inject the `light` CSS before the others, with
   // no media filter so that it will be downloaded with highest priority.
   if (window.matchMedia("(prefers-color-scheme: dark)").media === "not all") {
     document.documentElement.style.display = "none";
     document.head.insertAdjacentHTML(
       "beforeend",
       "<link id=\"css\" rel=\"stylesheet\" href=\"https://bootswatch.com/5/flatly/bootstrap.css\" onload=\"document.documentElement.style.display = ''\">"
     );
   }
 </script><title>SparkR - Practical Guide • SparkR</title>
 <script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 <link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet">
 <script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
 <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
 <!-- bootstrap-toc --><script src="https://cdn.rawgit.com/afeld/bootstrap-toc/v1.0.1/dist/bootstrap-toc.min.js"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script><!-- search --><script src="https://cdnjs.cloudflare.com/ajax/libs/fuse.js/6.4.6/fuse.js" integrity="sha512-zv6Ywkjyktsohkbp9bb45V6tEMoWhzFzXis+LrMehmJZZSys19Yxf1dopHx7WzIKxr5tK2dVcYmaCk2uqdjF4A==" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/autocomplete.js/0.38.0/autocomplete.jquery.min.js" integrity="sha512-GU9ayf+66Xx2TmpxqJpliWbT5PiGYxpaG8rfnBEk1LL8l1KGkRShhngwdXK1UgqhAzWpZHSiYPc09/NwDQIGyg==" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mark.js/8.11.1/mark.min.js" integrity="sha512-5CYOlHXGh6QpOFA/TeTylKLWfB3ftPsde7AnmhuitiTX4K5SqCLBeKro6sPS8ilsz1Q4NRx3v8Ko2IBiszzdww==" crossorigin="anonymous"></script><!-- pkgdown --><script src="../pkgdown.js"></script><link href="../extra.css" rel="stylesheet">
 <meta property="og:title" content="SparkR - Practical Guide">
 <meta property="og:description" content="SparkR">
 <!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
 <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
 <![endif]--><!-- Flatly Theme - Light  --><link id="css-light" rel="stylesheet" href="https://bootswatch.com/5/flatly/bootstrap.css" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)">
 <!-- Darkly Theme - Dark --><link id="css-dark" rel="stylesheet" href="https://bootswatch.com/5/darkly/bootstrap.css" media="(prefers-color-scheme: dark)">
 <!-- preferably CSS --><link rel="stylesheet" href="../preferably.css">
 <link id="css-code-light" rel="stylesheet" href="../code-color-scheme-light.css" media="(prefers-color-scheme: light), (prefers-color-scheme: no-preference)">
 <link id="css-code-dark" rel="stylesheet" href="../code-color-scheme-dark.css" media="(prefers-color-scheme: dark)">
 <script src="../darkswitch.js"></script>
 </head>
 <body>
     <a href="#main" class="visually-hidden-focusable">Skip to contents</a>


     <nav class="navbar fixed-top navbar-dark navbar-expand-lg bg-primary"><div class="container">
     <a class="external-link navbar-brand" href="https://spark.apache.org/">
   <img src="https://spark.apache.org/images/spark-logo-rev.svg" alt="" max-height="100%"></a>

     <a class="navbar-brand me-2" href="../index.html">SparkR</a>

     <small class="nav-text text-muted me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="">4.0.0</small>


     <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
       <span class="navbar-toggler-icon"></span>
     </button>

     <div id="navbar" class="collapse navbar-collapse ms-2">
       <ul class="navbar-nav me-auto">
 <li class="nav-item">
   <a class="nav-link" href="../reference/index.html">Reference</a>
 </li>
 <li class="active nav-item dropdown">
   <a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button" aria-expanded="false" aria-haspopup="true" id="dropdown-articles">Articles</a>
   <div class="dropdown-menu" aria-labelledby="dropdown-articles">
     <a class="dropdown-item" href="../articles/sparkr-vignettes.html">SparkR - Practical Guide</a>
   </div>
 </li>
       </ul>
 <form class="form-inline my-2 my-lg-0" role="search">
         <input type="search" class="form-control me-sm-2" aria-label="Toggle navigation" name="search-input" data-search-index="../search.json" id="search-input" placeholder="Search for" autocomplete="off">
 </form>

       <ul class="navbar-nav">
 <li>
           <a class="external-link nav-link" id="css-toggle-btn" aria-label="github">
             <span class="fas fa fas fa-adjust fa-lg"></span>
           </a>
         </li>


       </ul>
 </div>


   </div>
 </nav><div class="container template-article">


 <script src="sparkr-vignettes_files/accessible-code-block-0.0.1/empty-anchor.js"></script><div class="row">
   <main id="main" class="col-md-9"><div class="page-header">
       <img src="" class="logo" alt=""><h1>SparkR - Practical Guide</h1>


       <div class="d-none name"><code>sparkr-vignettes.Rmd</code></div>
     </div>


 <!--
  Licensed to the Apache Software Foundation (ASF) under one or more
  contributor license agreements.  See the NOTICE file distributed with
  this work for additional information regarding copyright ownership.
  The ASF licenses this file to You under the Apache License, Version 2.0
  (the "License"); you may not use this file except in compliance with
  the License.  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 -->
 <div class="section level2">
 <h2 id="overview">Overview<a class="anchor" aria-label="anchor" href="#overview"></a>
 </h2>
 <p>SparkR is an R package that provides a light-weight frontend to use Apache Spark from R. With Spark 4.0.0, SparkR provides a distributed data frame implementation that supports data processing operations like selection, filtering, aggregation etc. and distributed machine learning using <a href="https://spark.apache.org/mllib/" class="external-link">MLlib</a>.</p>
 </div>
 <div class="section level2">
 <h2 id="getting-started">Getting Started<a class="anchor" aria-label="anchor" href="#getting-started"></a>
 </h2>
 <p>We begin with an example running on the local machine and provide an overview of the use of SparkR: data ingestion, data processing and machine learning.</p>
 <p>First, let’s load and attach the package.</p>
 <div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="kw"><a href="https://rdrr.io/r/base/library.html" class="external-link">library</a></span><span class="op">(</span><span class="va"><a href="https://www.apache.org" class="external-link">SparkR</a></span><span class="op">)</span></span></code></pre></div>
 <p><code>SparkSession</code> is the entry point into SparkR which connects your R program to a Spark cluster. You can create a <code>SparkSession</code> using <code>sparkR.session</code> and pass in options such as the application name, any Spark packages depended on, etc.</p>
 <p>We use default settings in which it runs in local mode. It auto downloads Spark package in the background if no previous installation is found. For more details about setup, see <a href="#SetupSparkSession">Spark Session</a>.</p>
 <div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/sparkR.session.html">sparkR.session</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## Java ref type org.apache.spark.sql.SparkSession id 1</span></span></code></pre>
 <p>The operations in SparkR are centered around an R class called <code>SparkDataFrame</code>. It is a distributed collection of data organized into named columns, which is conceptually equivalent to a table in a relational database or a data frame in R, but with richer optimizations under the hood.</p>
 <p><code>SparkDataFrame</code> can be constructed from a wide array of sources such as: structured data files, tables in Hive, external databases, or existing local R data frames. For example, we create a <code>SparkDataFrame</code> from a local R data frame,</p>
 <div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">cars</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/cbind.html" class="external-link">cbind</a></span><span class="op">(</span>model <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">rownames</a></span><span class="op">(</span><span class="va">mtcars</span><span class="op">)</span>, <span class="va">mtcars</span><span class="op">)</span></span>
 <span><span class="va">carsDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">cars</span><span class="op">)</span></span></code></pre></div>
 <p>We can view the first few rows of the <code>SparkDataFrame</code> by <code>head</code> or <code>showDF</code> function.</p>
 <div class="sourceCode" id="cb5"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">carsDF</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##               model  mpg cyl disp  hp drat    wt  qsec vs am gear carb</span></span>
 <span><span class="co">## 1         Mazda RX4 21.0   6  160 110 3.90 2.620 16.46  0  1    4    4</span></span>
 <span><span class="co">## 2     Mazda RX4 Wag 21.0   6  160 110 3.90 2.875 17.02  0  1    4    4</span></span>
 <span><span class="co">## 3        Datsun 710 22.8   4  108  93 3.85 2.320 18.61  1  1    4    1</span></span>
 <span><span class="co">## 4    Hornet 4 Drive 21.4   6  258 110 3.08 3.215 19.44  1  0    3    1</span></span>
 <span><span class="co">## 5 Hornet Sportabout 18.7   8  360 175 3.15 3.440 17.02  0  0    3    2</span></span>
 <span><span class="co">## 6           Valiant 18.1   6  225 105 2.76 3.460 20.22  1  0    3    1</span></span></code></pre>
 <p>Common data processing operations such as <code>filter</code> and <code>select</code> are supported on the <code>SparkDataFrame</code>.</p>
 <div class="sourceCode" id="cb7"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">carsSubDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="st">"model"</span>, <span class="st">"mpg"</span>, <span class="st">"hp"</span><span class="op">)</span></span>
 <span><span class="va">carsSubDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/filter.html">filter</a></span><span class="op">(</span><span class="va">carsSubDF</span>, <span class="va">carsSubDF</span><span class="op">$</span><span class="va">hp</span> <span class="op">&gt;=</span> <span class="fl">200</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">carsSubDF</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##                 model  mpg  hp</span></span>
 <span><span class="co">## 1          Duster 360 14.3 245</span></span>
 <span><span class="co">## 2  Cadillac Fleetwood 10.4 205</span></span>
 <span><span class="co">## 3 Lincoln Continental 10.4 215</span></span>
 <span><span class="co">## 4   Chrysler Imperial 14.7 230</span></span>
 <span><span class="co">## 5          Camaro Z28 13.3 245</span></span>
 <span><span class="co">## 6      Ford Pantera L 15.8 264</span></span></code></pre>
 <p>SparkR can use many common aggregation functions after grouping.</p>
 <div class="sourceCode" id="cb9"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">carsGPDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/summarize.html">summarize</a></span><span class="op">(</span><span class="fu"><a href="../reference/groupBy.html">groupBy</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">carsDF</span><span class="op">$</span><span class="va">gear</span><span class="op">)</span>, count <span class="op">=</span> <span class="fu"><a href="../reference/count.html">n</a></span><span class="op">(</span><span class="va">carsDF</span><span class="op">$</span><span class="va">gear</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">carsGPDF</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   gear count</span></span>
 <span><span class="co">## 1    4    12</span></span>
 <span><span class="co">## 2    3    15</span></span>
 <span><span class="co">## 3    5     5</span></span></code></pre>
 <p>The results <code>carsDF</code> and <code>carsSubDF</code> are <code>SparkDataFrame</code> objects. To convert back to R <code>data.frame</code>, we can use <code>collect</code>. <strong>Caution</strong>: This can cause your interactive environment to run out of memory, though, because <code><a href="../reference/collect.html">collect()</a></code> fetches the entire distributed <code>DataFrame</code> to your client, which is acting as a Spark driver.</p>
 <div class="sourceCode" id="cb11"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">carsGP</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/collect.html">collect</a></span><span class="op">(</span><span class="va">carsGPDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">carsGP</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] "data.frame"</span></span></code></pre>
 <p>SparkR supports a number of commonly used machine learning algorithms. Under the hood, SparkR uses MLlib to train the model. Users can call <code>summary</code> to print a summary of the fitted model, <code>predict</code> to make predictions on new data, and <code>write.ml</code>/<code>read.ml</code> to save/load fitted models.</p>
 <p>SparkR supports a subset of R formula operators for model fitting, including ‘~’, ‘.’, ‘:’, ‘+’, and ‘-‘. We use linear regression as an example.</p>
 <div class="sourceCode" id="cb13"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.glm.html">spark.glm</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">mpg</span> <span class="op">~</span> <span class="va">wt</span> <span class="op">+</span> <span class="va">cyl</span><span class="op">)</span></span></code></pre></div>
 <p>The result matches that returned by R <code>glm</code> function applied to the corresponding <code>data.frame</code> <code>mtcars</code> of <code>carsDF</code>. In fact, for Generalized Linear Model, we specifically expose <code>glm</code> for <code>SparkDataFrame</code> as well so that the above is equivalent to <code>model &lt;- glm(mpg ~ wt + cyl, data = carsDF)</code>.</p>
 <div class="sourceCode" id="cb14"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## </span></span>
 <span><span class="co">## Deviance Residuals: </span></span>
 <span><span class="co">## (Note: These are approximate quantiles with relative error &lt;= 0.01)</span></span>
 <span><span class="co">##     Min       1Q   Median       3Q      Max  </span></span>
 <span><span class="co">## -4.2893  -1.7085  -0.4713   1.5729   6.1004  </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Coefficients:</span></span>
 <span><span class="co">##              Estimate  Std. Error  t value    Pr(&gt;|t|)</span></span>
 <span><span class="co">## (Intercept)   39.6863     1.71498  23.1409  0.00000000</span></span>
 <span><span class="co">## wt            -3.1910     0.75691  -4.2158  0.00022202</span></span>
 <span><span class="co">## cyl           -1.5078     0.41469  -3.6360  0.00106428</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## (Dispersion parameter for gaussian family taken to be 6.592137)</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">##     Null deviance: 1126.05  on 31  degrees of freedom</span></span>
 <span><span class="co">## Residual deviance:  191.17  on 29  degrees of freedom</span></span>
 <span><span class="co">## AIC: 156</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Number of Fisher Scoring iterations: 1</span></span></code></pre>
 <p>The model can be saved by <code>write.ml</code> and loaded back using <code>read.ml</code>.</p>
 <div class="sourceCode" id="cb16"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/write.ml.html">write.ml</a></span><span class="op">(</span><span class="va">model</span>, path <span class="op">=</span> <span class="st">"/HOME/tmp/mlModel/glmModel"</span><span class="op">)</span></span></code></pre></div>
 <p>In the end, we can stop Spark Session by running</p>
 <div class="sourceCode" id="cb17"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/sparkR.session.stop.html">sparkR.session.stop</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
 </div>
 <div class="section level2">
 <h2 id="setup">Setup<a class="anchor" aria-label="anchor" href="#setup"></a>
 </h2>
 <div class="section level3">
 <h3 id="installation">Installation<a class="anchor" aria-label="anchor" href="#installation"></a>
 </h3>
 <p>Different from many other R packages, to use SparkR, you need an additional installation of Apache Spark. The Spark installation will be used to run a backend process that will compile and execute SparkR programs.</p>
 <p>After installing the SparkR package, you can call <code>sparkR.session</code> as explained in the previous section to start and it will check for the Spark installation. If you are working with SparkR from an interactive shell (e.g. R, RStudio) then Spark is downloaded and cached automatically if it is not found. Alternatively, we provide an easy-to-use function <code>install.spark</code> for running this manually. If you don’t have Spark installed on the computer, you may download it from <a href="https://spark.apache.org/downloads.html" class="external-link">Apache Spark Website</a>.</p>
 <div class="sourceCode" id="cb18"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/install.spark.html">install.spark</a></span><span class="op">(</span><span class="op">)</span></span></code></pre></div>
 <p>If you already have Spark installed, you don’t have to install again and can pass the <code>sparkHome</code> argument to <code>sparkR.session</code> to let SparkR know where the existing Spark installation is.</p>
 <div class="sourceCode" id="cb19"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/sparkR.session.html">sparkR.session</a></span><span class="op">(</span>sparkHome <span class="op">=</span> <span class="st">"/HOME/spark"</span><span class="op">)</span></span></code></pre></div>
 </div>
 <div class="section level3">
 <h3 id="SetupSparkSession">Spark Session<a class="anchor" aria-label="anchor" href="#SetupSparkSession"></a>
 </h3>
 <p>In addition to <code>sparkHome</code>, many other options can be specified in <code>sparkR.session</code>. For a complete list, see <a href="https://spark.apache.org/docs/latest/sparkr.html#starting-up-sparksession" class="external-link">Starting up: SparkSession</a> and <a href="https://spark.apache.org/docs/latest/api/R/reference/sparkR.session.html" class="external-link">SparkR API doc</a>.</p>
 <p>In particular, the following Spark driver properties can be set in <code>sparkConfig</code>.</p>
 <table class="table">
 <thead><tr class="header">
 <th>Property Name</th>
 <th>Property group</th>
 <th>spark-submit equivalent</th>
 </tr></thead>
 <tbody>
 <tr class="odd">
 <td><code>spark.driver.memory</code></td>
 <td>Application Properties</td>
 <td><code>--driver-memory</code></td>
 </tr>
 <tr class="even">
 <td><code>spark.driver.extraClassPath</code></td>
 <td>Runtime Environment</td>
 <td><code>--driver-class-path</code></td>
 </tr>
 <tr class="odd">
 <td><code>spark.driver.extraJavaOptions</code></td>
 <td>Runtime Environment</td>
 <td><code>--driver-java-options</code></td>
 </tr>
 <tr class="even">
 <td><code>spark.driver.extraLibraryPath</code></td>
 <td>Runtime Environment</td>
 <td><code>--driver-library-path</code></td>
 </tr>
 <tr class="odd">
 <td><code>spark.kerberos.keytab</code></td>
 <td>Application Properties</td>
 <td><code>--keytab</code></td>
 </tr>
 <tr class="even">
 <td><code>spark.kerberos.principal</code></td>
 <td>Application Properties</td>
 <td><code>--principal</code></td>
 </tr>
 </tbody>
 </table>
 <p><strong>For Windows users</strong>: Due to different file prefixes across operating systems, to avoid the issue of potential wrong prefix, a current workaround is to specify <code>spark.sql.warehouse.dir</code> when starting the <code>SparkSession</code>.</p>
 <div class="sourceCode" id="cb20"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">spark_warehouse_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/path.expand.html" class="external-link">path.expand</a></span><span class="op">(</span><span class="st">'~'</span><span class="op">)</span>, <span class="st">"spark-warehouse"</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/sparkR.session.html">sparkR.session</a></span><span class="op">(</span>spark.sql.warehouse.dir <span class="op">=</span> <span class="va">spark_warehouse_path</span><span class="op">)</span></span></code></pre></div>
 <div class="section level4">
 <h4 id="cluster-mode">Cluster Mode<a class="anchor" aria-label="anchor" href="#cluster-mode"></a>
 </h4>
 <p>SparkR can connect to remote Spark clusters. <a href="https://spark.apache.org/docs/latest/cluster-overview.html" class="external-link">Cluster Mode Overview</a> is a good introduction to different Spark cluster modes.</p>
 <p>When connecting SparkR to a remote Spark cluster, make sure that the Spark version and Hadoop version on the machine match the corresponding versions on the cluster. Current SparkR package is compatible with</p>
 <pre><code><span><span class="co">## [1] "Spark 4.0.0"</span></span></code></pre>
 <p>It should be used both on the local computer and on the remote cluster.</p>
 <p>To connect, pass the URL of the master node to <code>sparkR.session</code>. A complete list can be seen in <a href="https://spark.apache.org/docs/latest/submitting-applications.html#master-urls" class="external-link">Spark Master URLs</a>. For example, to connect to a local standalone Spark master, we can call</p>
 <div class="sourceCode" id="cb22"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/sparkR.session.html">sparkR.session</a></span><span class="op">(</span>master <span class="op">=</span> <span class="st">"spark://local:7077"</span><span class="op">)</span></span></code></pre></div>
 <p>For YARN cluster, SparkR supports the client mode with the master set as “yarn”.</p>
 <div class="sourceCode" id="cb23"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/sparkR.session.html">sparkR.session</a></span><span class="op">(</span>master <span class="op">=</span> <span class="st">"yarn"</span><span class="op">)</span></span></code></pre></div>
 <p>Yarn cluster mode is not supported in the current version.</p>
 </div>
 </div>
 </div>
 <div class="section level2">
 <h2 id="data-import">Data Import<a class="anchor" aria-label="anchor" href="#data-import"></a>
 </h2>
 <div class="section level3">
 <h3 id="local-data-frame">Local Data Frame<a class="anchor" aria-label="anchor" href="#local-data-frame"></a>
 </h3>
 <p>The simplest way is to convert a local R data frame into a <code>SparkDataFrame</code>. Specifically we can use <code>as.DataFrame</code> or <code>createDataFrame</code> and pass in the local R data frame to create a <code>SparkDataFrame</code>. As an example, the following creates a <code>SparkDataFrame</code> based using the <code>faithful</code> dataset from R.</p>
 <div class="sourceCode" id="cb24"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">as.DataFrame</a></span><span class="op">(</span><span class="va">faithful</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">df</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   eruptions waiting</span></span>
 <span><span class="co">## 1     3.600      79</span></span>
 <span><span class="co">## 2     1.800      54</span></span>
 <span><span class="co">## 3     3.333      74</span></span>
 <span><span class="co">## 4     2.283      62</span></span>
 <span><span class="co">## 5     4.533      85</span></span>
 <span><span class="co">## 6     2.883      55</span></span></code></pre>
 </div>
 <div class="section level3">
 <h3 id="data-sources">Data Sources<a class="anchor" aria-label="anchor" href="#data-sources"></a>
 </h3>
 <p>SparkR supports operating on a variety of data sources through the <code>SparkDataFrame</code> interface. You can check the Spark SQL Programming Guide for more <a href="https://spark.apache.org/docs/latest/sql-programming-guide.html#manually-specifying-options" class="external-link">specific options</a> that are available for the built-in data sources.</p>
 <p>The general method for creating <code>SparkDataFrame</code> from data sources is <code>read.df</code>. This method takes in the path for the file to load and the type of data source, and the currently active Spark Session will be used automatically. SparkR supports reading CSV, JSON and Parquet files natively and through Spark Packages you can find data source connectors for popular file formats like Avro. These packages can be added with <code>sparkPackages</code> parameter when initializing SparkSession using <code>sparkR.session</code>.</p>
 <div class="sourceCode" id="cb26"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/sparkR.session.html">sparkR.session</a></span><span class="op">(</span>sparkPackages <span class="op">=</span> <span class="st">"com.databricks:spark-avro_2.12:3.0.0"</span><span class="op">)</span></span></code></pre></div>
 <p>We can see how to use data sources using an example CSV input file. For more information please refer to SparkR <a href="https://spark.apache.org/docs/latest/api/R/reference/read.df.html" class="external-link">read.df</a> API documentation.</p>
 <div class="sourceCode" id="cb27"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/read.df.html">read.df</a></span><span class="op">(</span><span class="va">csvPath</span>, <span class="st">"csv"</span>, header <span class="op">=</span> <span class="st">"true"</span>, inferSchema <span class="op">=</span> <span class="st">"true"</span>, na.strings <span class="op">=</span> <span class="st">"NA"</span><span class="op">)</span></span></code></pre></div>
 <p>The data sources API natively supports JSON formatted input files. Note that the file that is used here is not a typical JSON file. Each line in the file must contain a separate, self-contained valid JSON object. As a consequence, a regular multi-line JSON file will most often fail.</p>
 <p>Let’s take a look at the first two lines of the raw JSON file used here.</p>
 <div class="sourceCode" id="cb28"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">filePath</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="fu"><a href="../reference/sparkR.conf.html">sparkR.conf</a></span><span class="op">(</span><span class="st">"spark.home"</span><span class="op">)</span>,</span>
 <span>                         <span class="st">"/examples/src/main/resources/people.json"</span><span class="op">)</span></span>
 <span><span class="fu"><a href="https://rdrr.io/r/base/readLines.html" class="external-link">readLines</a></span><span class="op">(</span><span class="va">filePath</span>, n <span class="op">=</span> <span class="fl">2L</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] "{\"name\":\"Michael\"}"          "{\"name\":\"Andy\", \"age\":30}"</span></span></code></pre>
 <p>We use <code>read.df</code> to read that into a <code>SparkDataFrame</code>.</p>
 <div class="sourceCode" id="cb30"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">people</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/read.df.html">read.df</a></span><span class="op">(</span><span class="va">filePath</span>, <span class="st">"json"</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/count.html">count</a></span><span class="op">(</span><span class="va">people</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] 3</span></span></code></pre>
 <div class="sourceCode" id="cb32"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">people</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   age    name</span></span>
 <span><span class="co">## 1  NA Michael</span></span>
 <span><span class="co">## 2  30    Andy</span></span>
 <span><span class="co">## 3  19  Justin</span></span></code></pre>
 <p>SparkR automatically infers the schema from the JSON file.</p>
 <div class="sourceCode" id="cb34"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/printSchema.html">printSchema</a></span><span class="op">(</span><span class="va">people</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## root</span></span>
 <span><span class="co">##  |-- age: long (nullable = true)</span></span>
 <span><span class="co">##  |-- name: string (nullable = true)</span></span></code></pre>
 <p>If we want to read multiple JSON files, <code>read.json</code> can be used.</p>
 <div class="sourceCode" id="cb36"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">people</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/read.json.html">read.json</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/Sys.getenv.html" class="external-link">Sys.getenv</a></span><span class="op">(</span><span class="st">"SPARK_HOME"</span><span class="op">)</span>,</span>
 <span>                           <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"/examples/src/main/resources/people.json"</span>,</span>
 <span>                             <span class="st">"/examples/src/main/resources/people.json"</span><span class="op">)</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/count.html">count</a></span><span class="op">(</span><span class="va">people</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] 6</span></span></code></pre>
 <p>The data sources API can also be used to save out <code>SparkDataFrames</code> into multiple file formats. For example we can save the <code>SparkDataFrame</code> from the previous example to a Parquet file using <code>write.df</code>.</p>
 <div class="sourceCode" id="cb38"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/write.df.html">write.df</a></span><span class="op">(</span><span class="va">people</span>, path <span class="op">=</span> <span class="st">"people.parquet"</span>, source <span class="op">=</span> <span class="st">"parquet"</span>, mode <span class="op">=</span> <span class="st">"overwrite"</span><span class="op">)</span></span></code></pre></div>
 </div>
 <div class="section level3">
 <h3 id="hive-tables">Hive Tables<a class="anchor" aria-label="anchor" href="#hive-tables"></a>
 </h3>
 <p>You can also create SparkDataFrames from Hive tables. To do this we will need to create a SparkSession with Hive support which can access tables in the Hive MetaStore. Note that Spark should have been built with Hive support and more details can be found in the <a href="https://spark.apache.org/docs/latest/sql-programming-guide.html" class="external-link">SQL Programming Guide</a>. In SparkR, by default it will attempt to create a SparkSession with Hive support enabled (<code>enableHiveSupport = TRUE</code>).</p>
 <div class="sourceCode" id="cb39"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/sql.html">sql</a></span><span class="op">(</span><span class="st">"CREATE TABLE IF NOT EXISTS src (key INT, value STRING)"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="va">txtPath</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="fu"><a href="../reference/sparkR.conf.html">sparkR.conf</a></span><span class="op">(</span><span class="st">"spark.home"</span><span class="op">)</span>, <span class="st">"/examples/src/main/resources/kv1.txt"</span><span class="op">)</span></span>
 <span><span class="va">sqlCMD</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/sprintf.html" class="external-link">sprintf</a></span><span class="op">(</span><span class="st">"LOAD DATA LOCAL INPATH '%s' INTO TABLE src"</span>, <span class="va">txtPath</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/sql.html">sql</a></span><span class="op">(</span><span class="va">sqlCMD</span><span class="op">)</span></span>
 <span></span>
 <span><span class="va">results</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/sql.html">sql</a></span><span class="op">(</span><span class="st">"FROM src SELECT key, value"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># results is now a SparkDataFrame</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">results</span><span class="op">)</span></span></code></pre></div>
 </div>
 </div>
 <div class="section level2">
 <h2 id="data-processing">Data Processing<a class="anchor" aria-label="anchor" href="#data-processing"></a>
 </h2>
 <p><strong>To dplyr users</strong>: SparkR has similar interface as dplyr in data processing. However, some noticeable differences are worth mentioning in the first place. We use <code>df</code> to represent a <code>SparkDataFrame</code> and <code>col</code> to represent the name of column here.</p>
 <ol style="list-style-type: decimal">
 <li><p>indicate columns. SparkR uses either a character string of the column name or a Column object constructed with <code>$</code> to indicate a column. For example, to select <code>col</code> in <code>df</code>, we can write <code>select(df, "col")</code> or <code>select(df, df$col)</code>.</p></li>
 <li><p>describe conditions. In SparkR, the Column object representation can be inserted into the condition directly, or we can use a character string to describe the condition, without referring to the <code>SparkDataFrame</code> used. For example, to select rows with value &gt; 1, we can write <code>filter(df, df$col &gt; 1)</code> or <code>filter(df, "col &gt; 1")</code>.</p></li>
 </ol>
 <p>Here are more concrete examples.</p>
 <table class="table">
 <colgroup>
 <col width="47%">
 <col width="52%">
 </colgroup>
 <thead><tr class="header">
 <th>dplyr</th>
 <th>SparkR</th>
 </tr></thead>
 <tbody>
 <tr class="odd">
 <td><code>select(mtcars, mpg, hp)</code></td>
 <td><code>select(carsDF, "mpg", "hp")</code></td>
 </tr>
 <tr class="even">
 <td><code>filter(mtcars, mpg &gt; 20, hp &gt; 100)</code></td>
 <td><code>filter(carsDF, carsDF$mpg &gt; 20, carsDF$hp &gt; 100)</code></td>
 </tr>
 </tbody>
 </table>
 <p>Other differences will be mentioned in the specific methods.</p>
 <p>We use the <code>SparkDataFrame</code> <code>carsDF</code> created above. We can get basic information about the <code>SparkDataFrame</code>.</p>
 <div class="sourceCode" id="cb40"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">carsDF</span></span></code></pre></div>
 <pre><code><span><span class="co">## SparkDataFrame[model:string, mpg:double, cyl:double, disp:double, hp:double, drat:double, wt:double, qsec:double, vs:double, am:double, gear:double, carb:double]</span></span></code></pre>
 <p>Print out the schema in tree format.</p>
 <div class="sourceCode" id="cb42"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/printSchema.html">printSchema</a></span><span class="op">(</span><span class="va">carsDF</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## root</span></span>
 <span><span class="co">##  |-- model: string (nullable = true)</span></span>
 <span><span class="co">##  |-- mpg: double (nullable = true)</span></span>
 <span><span class="co">##  |-- cyl: double (nullable = true)</span></span>
 <span><span class="co">##  |-- disp: double (nullable = true)</span></span>
 <span><span class="co">##  |-- hp: double (nullable = true)</span></span>
 <span><span class="co">##  |-- drat: double (nullable = true)</span></span>
 <span><span class="co">##  |-- wt: double (nullable = true)</span></span>
 <span><span class="co">##  |-- qsec: double (nullable = true)</span></span>
 <span><span class="co">##  |-- vs: double (nullable = true)</span></span>
 <span><span class="co">##  |-- am: double (nullable = true)</span></span>
 <span><span class="co">##  |-- gear: double (nullable = true)</span></span>
 <span><span class="co">##  |-- carb: double (nullable = true)</span></span></code></pre>
 <div class="section level3">
 <h3 id="sparkdataframe-operations">SparkDataFrame Operations<a class="anchor" aria-label="anchor" href="#sparkdataframe-operations"></a>
 </h3>
 <div class="section level4">
 <h4 id="selecting-rows-columns">Selecting rows, columns<a class="anchor" aria-label="anchor" href="#selecting-rows-columns"></a>
 </h4>
 <p>SparkDataFrames support a number of functions to do structured data processing. Here we include some basic examples and a complete list can be found in the <a href="https://spark.apache.org/docs/latest/api/R/index.html" class="external-link">API</a> docs:</p>
 <p>You can also pass in column name as strings.</p>
 <div class="sourceCode" id="cb44"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="st">"mpg"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##    mpg</span></span>
 <span><span class="co">## 1 21.0</span></span>
 <span><span class="co">## 2 21.0</span></span>
 <span><span class="co">## 3 22.8</span></span>
 <span><span class="co">## 4 21.4</span></span>
 <span><span class="co">## 5 18.7</span></span>
 <span><span class="co">## 6 18.1</span></span></code></pre>
 <p>Filter the SparkDataFrame to only retain rows with mpg less than 20 miles/gallon.</p>
 <div class="sourceCode" id="cb46"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/filter.html">filter</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">carsDF</span><span class="op">$</span><span class="va">mpg</span> <span class="op">&lt;</span> <span class="fl">20</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##               model  mpg cyl  disp  hp drat   wt  qsec vs am gear carb</span></span>
 <span><span class="co">## 1 Hornet Sportabout 18.7   8 360.0 175 3.15 3.44 17.02  0  0    3    2</span></span>
 <span><span class="co">## 2           Valiant 18.1   6 225.0 105 2.76 3.46 20.22  1  0    3    1</span></span>
 <span><span class="co">## 3        Duster 360 14.3   8 360.0 245 3.21 3.57 15.84  0  0    3    4</span></span>
 <span><span class="co">## 4          Merc 280 19.2   6 167.6 123 3.92 3.44 18.30  1  0    4    4</span></span>
 <span><span class="co">## 5         Merc 280C 17.8   6 167.6 123 3.92 3.44 18.90  1  0    4    4</span></span>
 <span><span class="co">## 6        Merc 450SE 16.4   8 275.8 180 3.07 4.07 17.40  0  0    3    3</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="grouping-aggregation">Grouping, Aggregation<a class="anchor" aria-label="anchor" href="#grouping-aggregation"></a>
 </h4>
 <p>A common flow of grouping and aggregation is</p>
 <ol style="list-style-type: decimal">
 <li><p>Use <code>groupBy</code> or <code>group_by</code> with respect to some grouping variables to create a <code>GroupedData</code> object</p></li>
 <li><p>Feed the <code>GroupedData</code> object to <code>agg</code> or <code>summarize</code> functions, with some provided aggregation functions to compute a number within each group.</p></li>
 </ol>
 <p>A number of widely used functions are supported to aggregate data after grouping, including <code>avg</code>, <code>count_distinct</code>, <code>count</code>, <code>first</code>, <code>kurtosis</code>, <code>last</code>, <code>max</code>, <code>mean</code>, <code>min</code>, <code>sd</code>, <code>skewness</code>, <code>stddev_pop</code>, <code>stddev_samp</code>, <code>sum_distinct</code>, <code>sum</code>, <code>var_pop</code>, <code>var_samp</code>, <code>var</code>. See the <a href="https://spark.apache.org/docs/latest/api/R/reference/column_aggregate_functions.html" class="external-link">API doc for aggregate functions</a> linked there.</p>
 <p>For example we can compute a histogram of the number of cylinders in the <code>mtcars</code> dataset as shown below.</p>
 <div class="sourceCode" id="cb48"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">numCyl</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/summarize.html">summarize</a></span><span class="op">(</span><span class="fu"><a href="../reference/groupBy.html">groupBy</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">carsDF</span><span class="op">$</span><span class="va">cyl</span><span class="op">)</span>, count <span class="op">=</span> <span class="fu"><a href="../reference/count.html">n</a></span><span class="op">(</span><span class="va">carsDF</span><span class="op">$</span><span class="va">cyl</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">numCyl</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   cyl count</span></span>
 <span><span class="co">## 1   8    14</span></span>
 <span><span class="co">## 2   4    11</span></span>
 <span><span class="co">## 3   6     7</span></span></code></pre>
 <p>Use <code>cube</code> or <code>rollup</code> to compute subtotals across multiple dimensions.</p>
 <div class="sourceCode" id="cb50"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/column_aggregate_functions.html">mean</a></span><span class="op">(</span><span class="fu"><a href="../reference/cube.html">cube</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="st">"cyl"</span>, <span class="st">"gear"</span>, <span class="st">"am"</span><span class="op">)</span>, <span class="st">"mpg"</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## SparkDataFrame[cyl:double, gear:double, am:double, avg(mpg):double]</span></span></code></pre>
 <p>generates groupings for {(<code>cyl</code>, <code>gear</code>, <code>am</code>), (<code>cyl</code>, <code>gear</code>), (<code>cyl</code>), ()}, while</p>
 <div class="sourceCode" id="cb52"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/column_aggregate_functions.html">mean</a></span><span class="op">(</span><span class="fu"><a href="../reference/rollup.html">rollup</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="st">"cyl"</span>, <span class="st">"gear"</span>, <span class="st">"am"</span><span class="op">)</span>, <span class="st">"mpg"</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## SparkDataFrame[cyl:double, gear:double, am:double, avg(mpg):double]</span></span></code></pre>
 <p>generates groupings for all possible combinations of grouping columns.</p>
 </div>
 <div class="section level4">
 <h4 id="operating-on-columns">Operating on Columns<a class="anchor" aria-label="anchor" href="#operating-on-columns"></a>
 </h4>
 <p>SparkR also provides a number of functions that can directly applied to columns for data processing and during aggregation. The example below shows the use of basic arithmetic functions.</p>
 <div class="sourceCode" id="cb54"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">carsDF_km</span> <span class="op">&lt;-</span> <span class="va">carsDF</span></span>
 <span><span class="va">carsDF_km</span><span class="op">$</span><span class="va">kmpg</span> <span class="op">&lt;-</span> <span class="va">carsDF_km</span><span class="op">$</span><span class="va">mpg</span> <span class="op">*</span> <span class="fl">1.61</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">carsDF_km</span>, <span class="st">"model"</span>, <span class="st">"mpg"</span>, <span class="st">"kmpg"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##               model  mpg   kmpg</span></span>
 <span><span class="co">## 1         Mazda RX4 21.0 33.810</span></span>
 <span><span class="co">## 2     Mazda RX4 Wag 21.0 33.810</span></span>
 <span><span class="co">## 3        Datsun 710 22.8 36.708</span></span>
 <span><span class="co">## 4    Hornet 4 Drive 21.4 34.454</span></span>
 <span><span class="co">## 5 Hornet Sportabout 18.7 30.107</span></span>
 <span><span class="co">## 6           Valiant 18.1 29.141</span></span></code></pre>
 </div>
 </div>
 <div class="section level3">
 <h3 id="window-functions">Window Functions<a class="anchor" aria-label="anchor" href="#window-functions"></a>
 </h3>
 <p>A window function is a variation of aggregation function. In simple words,</p>
 <ul>
 <li><p>aggregation function: <code>n</code> to <code>1</code> mapping - returns a single value for a group of entries. Examples include <code>sum</code>, <code>count</code>, <code>max</code>.</p></li>
 <li><p>window function: <code>n</code> to <code>n</code> mapping - returns one value for each entry in the group, but the value may depend on all the entries of the <em>group</em>. Examples include <code>rank</code>, <code>lead</code>, <code>lag</code>.</p></li>
 </ul>
 <p>Formally, the <em>group</em> mentioned above is called the <em>frame</em>. Every input row can have a unique frame associated with it and the output of the window function on that row is based on the rows confined in that frame.</p>
 <p>Window functions are often used in conjunction with the following functions: <code>windowPartitionBy</code>, <code>windowOrderBy</code>, <code>partitionBy</code>, <code>orderBy</code>, <code>over</code>. To illustrate this we next look at an example.</p>
 <p>We still use the <code>mtcars</code> dataset. The corresponding <code>SparkDataFrame</code> is <code>carsDF</code>. Suppose for each number of cylinders, we want to calculate the rank of each car in <code>mpg</code> within the group.</p>
 <div class="sourceCode" id="cb56"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">carsSubDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="st">"model"</span>, <span class="st">"mpg"</span>, <span class="st">"cyl"</span><span class="op">)</span></span>
 <span><span class="va">ws</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/orderBy.html">orderBy</a></span><span class="op">(</span><span class="fu"><a href="../reference/windowPartitionBy.html">windowPartitionBy</a></span><span class="op">(</span><span class="st">"cyl"</span><span class="op">)</span>, <span class="st">"mpg"</span><span class="op">)</span></span>
 <span><span class="va">carsRank</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/withColumn.html">withColumn</a></span><span class="op">(</span><span class="va">carsSubDF</span>, <span class="st">"rank"</span>, <span class="fu"><a href="../reference/over.html">over</a></span><span class="op">(</span><span class="fu"><a href="../reference/column_window_functions.html">rank</a></span><span class="op">(</span><span class="op">)</span>, <span class="va">ws</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">carsRank</span>, n <span class="op">=</span> <span class="fl">20L</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##                  model  mpg cyl rank</span></span>
 <span><span class="co">## 1           Volvo 142E 21.4   4    1</span></span>
 <span><span class="co">## 2        Toyota Corona 21.5   4    2</span></span>
 <span><span class="co">## 3           Datsun 710 22.8   4    3</span></span>
 <span><span class="co">## 4             Merc 230 22.8   4    3</span></span>
 <span><span class="co">## 5            Merc 240D 24.4   4    5</span></span>
 <span><span class="co">## 6        Porsche 914-2 26.0   4    6</span></span>
 <span><span class="co">## 7            Fiat X1-9 27.3   4    7</span></span>
 <span><span class="co">## 8          Honda Civic 30.4   4    8</span></span>
 <span><span class="co">## 9         Lotus Europa 30.4   4    8</span></span>
 <span><span class="co">## 10            Fiat 128 32.4   4   10</span></span>
 <span><span class="co">## 11      Toyota Corolla 33.9   4   11</span></span>
 <span><span class="co">## 12           Merc 280C 17.8   6    1</span></span>
 <span><span class="co">## 13             Valiant 18.1   6    2</span></span>
 <span><span class="co">## 14            Merc 280 19.2   6    3</span></span>
 <span><span class="co">## 15        Ferrari Dino 19.7   6    4</span></span>
 <span><span class="co">## 16           Mazda RX4 21.0   6    5</span></span>
 <span><span class="co">## 17       Mazda RX4 Wag 21.0   6    5</span></span>
 <span><span class="co">## 18      Hornet 4 Drive 21.4   6    7</span></span>
 <span><span class="co">## 19  Cadillac Fleetwood 10.4   8    1</span></span>
 <span><span class="co">## 20 Lincoln Continental 10.4   8    1</span></span></code></pre>
 <p>We explain in detail the above steps.</p>
 <ul>
 <li>
 <code>windowPartitionBy</code> creates a window specification object <code>WindowSpec</code> that defines the partition. It controls which rows will be in the same partition as the given row. In this case, rows with the same value in <code>cyl</code> will be put in the same partition. <code>orderBy</code> further defines the ordering - the position a given row is in the partition. The resulting <code>WindowSpec</code> is returned as <code>ws</code>.</li>
 </ul>
 <p>More window specification methods include <code>rangeBetween</code>, which can define boundaries of the frame by value, and <code>rowsBetween</code>, which can define the boundaries by row indices.</p>
 <ul>
 <li>
 <code>withColumn</code> appends a Column called <code>rank</code> to the <code>SparkDataFrame</code>. <code>over</code> returns a windowing column. The first argument is usually a Column returned by window function(s) such as <code><a href="../reference/column_window_functions.html">rank()</a></code>, <code>lead(carsDF$wt)</code>. That calculates the corresponding values according to the partitioned-and-ordered table.</li>
 </ul>
 </div>
 <div class="section level3">
 <h3 id="user-defined-function">User-Defined Function<a class="anchor" aria-label="anchor" href="#user-defined-function"></a>
 </h3>
 <p>In SparkR, we support several kinds of user-defined functions (UDFs).</p>
 <div class="section level4">
 <h4 id="apply-by-partition">Apply by Partition<a class="anchor" aria-label="anchor" href="#apply-by-partition"></a>
 </h4>
 <p><code>dapply</code> can apply a function to each partition of a <code>SparkDataFrame</code>. The function to be applied to each partition of the <code>SparkDataFrame</code> should have only one parameter, a <code>data.frame</code> corresponding to a partition, and the output should be a <code>data.frame</code> as well. Schema specifies the row format of the resulting a <code>SparkDataFrame</code>. It must match to data types of returned value. See <a href="#DataTypes">here</a> for mapping between R and Spark.</p>
 <p>We convert <code>mpg</code> to <code>kmpg</code> (kilometers per gallon). <code>carsSubDF</code> is a <code>SparkDataFrame</code> with a subset of <code>carsDF</code> columns.</p>
 <div class="sourceCode" id="cb58"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">carsSubDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="st">"model"</span>, <span class="st">"mpg"</span><span class="op">)</span></span>
 <span><span class="va">schema</span> <span class="op">&lt;-</span> <span class="st">"model STRING, mpg DOUBLE, kmpg DOUBLE"</span></span>
 <span><span class="va">out</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/dapply.html">dapply</a></span><span class="op">(</span><span class="va">carsSubDF</span>, <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">{</span> <span class="va">x</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/cbind.html" class="external-link">cbind</a></span><span class="op">(</span><span class="va">x</span>, <span class="va">x</span><span class="op">$</span><span class="va">mpg</span> <span class="op">*</span> <span class="fl">1.61</span><span class="op">)</span> <span class="op">}</span>, <span class="va">schema</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/collect.html">collect</a></span><span class="op">(</span><span class="va">out</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##               model  mpg   kmpg</span></span>
 <span><span class="co">## 1         Mazda RX4 21.0 33.810</span></span>
 <span><span class="co">## 2     Mazda RX4 Wag 21.0 33.810</span></span>
 <span><span class="co">## 3        Datsun 710 22.8 36.708</span></span>
 <span><span class="co">## 4    Hornet 4 Drive 21.4 34.454</span></span>
 <span><span class="co">## 5 Hornet Sportabout 18.7 30.107</span></span>
 <span><span class="co">## 6           Valiant 18.1 29.141</span></span></code></pre>
 <p>Like <code>dapply</code>, <code>dapplyCollect</code> can apply a function to each partition of a <code>SparkDataFrame</code> and collect the result back. The output of the function should be a <code>data.frame</code>, but no schema is required in this case. Note that <code>dapplyCollect</code> can fail if the output of the UDF on all partitions cannot be pulled into the driver’s memory.</p>
 <div class="sourceCode" id="cb60"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">out</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/dapplyCollect.html">dapplyCollect</a></span><span class="op">(</span></span>
 <span>         <span class="va">carsSubDF</span>,</span>
 <span>         <span class="kw">function</span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">{</span></span>
 <span>           <span class="va">x</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/cbind.html" class="external-link">cbind</a></span><span class="op">(</span><span class="va">x</span>, <span class="st">"kmpg"</span> <span class="op">=</span> <span class="va">x</span><span class="op">$</span><span class="va">mpg</span> <span class="op">*</span> <span class="fl">1.61</span><span class="op">)</span></span>
 <span>         <span class="op">}</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">out</span>, <span class="fl">3</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##           model  mpg   kmpg</span></span>
 <span><span class="co">## 1     Mazda RX4 21.0 33.810</span></span>
 <span><span class="co">## 2 Mazda RX4 Wag 21.0 33.810</span></span>
 <span><span class="co">## 3    Datsun 710 22.8 36.708</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="apply-by-group">Apply by Group<a class="anchor" aria-label="anchor" href="#apply-by-group"></a>
 </h4>
 <p><code>gapply</code> can apply a function to each group of a <code>SparkDataFrame</code>. The function is to be applied to each group of the <code>SparkDataFrame</code> and should have only two parameters: grouping key and R <code>data.frame</code> corresponding to that key. The groups are chosen from <code>SparkDataFrames</code> column(s). The output of function should be a <code>data.frame</code>. Schema specifies the row format of the resulting <code>SparkDataFrame</code>. It must represent R function’s output schema on the basis of Spark data types. The column names of the returned <code>data.frame</code> are set by user. See <a href="#DataTypes">here</a> for mapping between R and Spark.</p>
 <div class="sourceCode" id="cb62"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">schema</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/structType.html">structType</a></span><span class="op">(</span><span class="fu"><a href="../reference/structField.html">structField</a></span><span class="op">(</span><span class="st">"cyl"</span>, <span class="st">"double"</span><span class="op">)</span>, <span class="fu"><a href="../reference/structField.html">structField</a></span><span class="op">(</span><span class="st">"max_mpg"</span>, <span class="st">"double"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/gapply.html">gapply</a></span><span class="op">(</span></span>
 <span>    <span class="va">carsDF</span>,</span>
 <span>    <span class="st">"cyl"</span>,</span>
 <span>    <span class="kw">function</span><span class="op">(</span><span class="va">key</span>, <span class="va">x</span><span class="op">)</span> <span class="op">{</span></span>
 <span>        <span class="va">y</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span><span class="va">key</span>, <span class="fu"><a href="../reference/column_aggregate_functions.html">max</a></span><span class="op">(</span><span class="va">x</span><span class="op">$</span><span class="va">mpg</span><span class="op">)</span><span class="op">)</span></span>
 <span>    <span class="op">}</span>,</span>
 <span>    <span class="va">schema</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/arrange.html">arrange</a></span><span class="op">(</span><span class="va">result</span>, <span class="st">"max_mpg"</span>, decreasing <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   cyl max_mpg</span></span>
 <span><span class="co">## 1   4    33.9</span></span>
 <span><span class="co">## 2   6    21.4</span></span>
 <span><span class="co">## 3   8    19.2</span></span></code></pre>
 <p>Like <code>gapply</code>, <code>gapplyCollect</code> can apply a function to each partition of a <code>SparkDataFrame</code> and collect the result back to R <code>data.frame</code>. The output of the function should be a <code>data.frame</code> but no schema is required in this case. Note that <code>gapplyCollect</code> can fail if the output of the UDF on all partitions cannot be pulled into the driver’s memory.</p>
 <div class="sourceCode" id="cb64"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">result</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/gapplyCollect.html">gapplyCollect</a></span><span class="op">(</span></span>
 <span>    <span class="va">carsDF</span>,</span>
 <span>    <span class="st">"cyl"</span>,</span>
 <span>    <span class="kw">function</span><span class="op">(</span><span class="va">key</span>, <span class="va">x</span><span class="op">)</span> <span class="op">{</span></span>
 <span>         <span class="va">y</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span><span class="va">key</span>, <span class="fu"><a href="../reference/column_aggregate_functions.html">max</a></span><span class="op">(</span><span class="va">x</span><span class="op">$</span><span class="va">mpg</span><span class="op">)</span><span class="op">)</span></span>
 <span>        <span class="fu"><a href="../reference/columns.html">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"cyl"</span>, <span class="st">"max_mpg"</span><span class="op">)</span></span>
 <span>        <span class="va">y</span></span>
 <span>    <span class="op">}</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">result</span><span class="op">[</span><span class="fu"><a href="https://rdrr.io/r/base/order.html" class="external-link">order</a></span><span class="op">(</span><span class="va">result</span><span class="op">$</span><span class="va">max_mpg</span>, decreasing <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span>, <span class="op">]</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   cyl max_mpg</span></span>
 <span><span class="co">## 1   4    33.9</span></span>
 <span><span class="co">## 2   6    21.4</span></span>
 <span><span class="co">## 3   8    19.2</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="distribute-local-functions">Distribute Local Functions<a class="anchor" aria-label="anchor" href="#distribute-local-functions"></a>
 </h4>
 <p>Similar to <code>lapply</code> in native R, <code>spark.lapply</code> runs a function over a list of elements and distributes the computations with Spark. <code>spark.lapply</code> works in a manner that is similar to <code>doParallel</code> or <code>lapply</code> to elements of a list. The results of all the computations should fit in a single machine. If that is not the case you can do something like <code>df &lt;- createDataFrame(list)</code> and then use <code>dapply</code>.</p>
 <p>We use <code>svm</code> in package <code>e1071</code> as an example. We use all default settings except for varying costs of constraints violation. <code>spark.lapply</code> can train those different models in parallel.</p>
 <div class="sourceCode" id="cb66"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">costs</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/column_math_functions.html">exp</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/seq.html" class="external-link">seq</a></span><span class="op">(</span>from <span class="op">=</span> <span class="fu"><a href="../reference/column_math_functions.html">log</a></span><span class="op">(</span><span class="fl">1</span><span class="op">)</span>, to <span class="op">=</span> <span class="fu"><a href="../reference/column_math_functions.html">log</a></span><span class="op">(</span><span class="fl">1000</span><span class="op">)</span>, length.out <span class="op">=</span> <span class="fl">5</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">train</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">cost</span><span class="op">)</span> <span class="op">{</span></span>
 <span>  <span class="fu"><a href="https://rdrr.io/r/base/stopifnot.html" class="external-link">stopifnot</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/ns-load.html" class="external-link">requireNamespace</a></span><span class="op">(</span><span class="st">"e1071"</span>, quietly <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span><span class="op">)</span></span>
 <span>  <span class="va">model</span> <span class="op">&lt;-</span> <span class="fu">e1071</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/e1071/man/svm.html" class="external-link">svm</a></span><span class="op">(</span><span class="va">Species</span> <span class="op">~</span> <span class="va">.</span>, data <span class="op">=</span> <span class="va">iris</span>, cost <span class="op">=</span> <span class="va">cost</span><span class="op">)</span></span>
 <span>  <span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span>
 <span><span class="op">}</span></span></code></pre></div>
 <p>Return a list of model’s summaries.</p>
 <div class="sourceCode" id="cb67"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">model.summaries</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.lapply.html">spark.lapply</a></span><span class="op">(</span><span class="va">costs</span>, <span class="va">train</span><span class="op">)</span></span></code></pre></div>
 <div class="sourceCode" id="cb68"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/class.html" class="external-link">class</a></span><span class="op">(</span><span class="va">model.summaries</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] "list"</span></span></code></pre>
 <p>To avoid lengthy display, we only present the partial result of the second fitted model. You are free to inspect other models as well.</p>
 <div class="sourceCode" id="cb70"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/print.html" class="external-link">print</a></span><span class="op">(</span><span class="va">model.summaries</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $call</span></span>
 <span><span class="co">## svm(formula = Species ~ ., data = iris, cost = cost)</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $type</span></span>
 <span><span class="co">## [1] 0</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $kernel</span></span>
 <span><span class="co">## [1] 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $cost</span></span>
 <span><span class="co">## [1] 5.623413</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $degree</span></span>
 <span><span class="co">## [1] 3</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $gamma</span></span>
 <span><span class="co">## [1] 0.25</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $coef0</span></span>
 <span><span class="co">## [1] 0</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $nu</span></span>
 <span><span class="co">## [1] 0.5</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $epsilon</span></span>
 <span><span class="co">## [1] 0.1</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $sparse</span></span>
 <span><span class="co">## [1] FALSE</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $scaled</span></span>
 <span><span class="co">## [1] TRUE TRUE TRUE TRUE</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $x.scale</span></span>
 <span><span class="co">## $x.scale$`scaled:center`</span></span>
 <span><span class="co">## Sepal.Length  Sepal.Width Petal.Length  Petal.Width </span></span>
 <span><span class="co">##     5.843333     3.057333     3.758000     1.199333 </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $x.scale$`scaled:scale`</span></span>
 <span><span class="co">## Sepal.Length  Sepal.Width Petal.Length  Petal.Width </span></span>
 <span><span class="co">##    0.8280661    0.4358663    1.7652982    0.7622377 </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $y.scale</span></span>
 <span><span class="co">## NULL</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $nclasses</span></span>
 <span><span class="co">## [1] 3</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $levels</span></span>
 <span><span class="co">## [1] "setosa"     "versicolor" "virginica" </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $tot.nSV</span></span>
 <span><span class="co">## [1] 35</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $nSV</span></span>
 <span><span class="co">## [1]  6 15 14</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $labels</span></span>
 <span><span class="co">## [1] 1 2 3</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $SV</span></span>
 <span><span class="co">##     Sepal.Length Sepal.Width Petal.Length Petal.Width</span></span>
 <span><span class="co">## 14   -1.86378030 -0.13153881   -1.5056946  -1.4422448</span></span>
 <span><span class="co">## 16   -0.17309407  3.08045544   -1.2791040  -1.0486668</span></span>
 <span><span class="co">## 21   -0.53538397  0.78617383   -1.1658087  -1.3110521</span></span>
 <span><span class="co">## 23   -1.50149039  1.24503015   -1.5623422  -1.3110521</span></span>
 <span><span class="co">## 24   -0.89767388  0.55674567   -1.1658087  -0.9174741</span></span>
 <span><span class="co">## 42   -1.62225369 -1.73753594   -1.3923993  -1.1798595</span></span>
 <span><span class="co">## 51    1.39682886  0.32731751    0.5336209   0.2632600</span></span>
 <span><span class="co">## 53    1.27606556  0.09788935    0.6469162   0.3944526</span></span>
 <span><span class="co">## 54   -0.41462067 -1.73753594    0.1370873   0.1320673</span></span>
 <span><span class="co">## 55    0.79301235 -0.59039513    0.4769732   0.3944526</span></span>
 <span><span class="co">##  [ reached getOption("max.print") -- omitted 25 rows ]</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $index</span></span>
 <span><span class="co">##  [1]  14  16  21  23  24  42  51  53  54  55  58  61  69  71  73  78  79  84  85</span></span>
 <span><span class="co">## [20]  86  99 107 111 119 120 124 127 128 130 132 134 135 139 149 150</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $rho</span></span>
 <span><span class="co">## [1] -0.10346530  0.12160294 -0.09540346</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $compprob</span></span>
 <span><span class="co">## [1] FALSE</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $probA</span></span>
 <span><span class="co">## NULL</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $probB</span></span>
 <span><span class="co">## NULL</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $sigma</span></span>
 <span><span class="co">## NULL</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $coefs</span></span>
 <span><span class="co">##              [,1]        [,2]</span></span>
 <span><span class="co">##  [1,]  0.00000000  0.06561739</span></span>
 <span><span class="co">##  [2,]  0.76813720  0.93378721</span></span>
 <span><span class="co">##  [3,]  0.00000000  0.12123270</span></span>
 <span><span class="co">##  [4,]  0.00000000  0.31170741</span></span>
 <span><span class="co">##  [5,]  1.11614066  0.46397392</span></span>
 <span><span class="co">##  [6,]  1.88141600  1.10392128</span></span>
 <span><span class="co">##  [7,] -0.55872622  0.00000000</span></span>
 <span><span class="co">##  [8,]  0.00000000  5.62341325</span></span>
 <span><span class="co">##  [9,]  0.00000000  0.27711792</span></span>
 <span><span class="co">## [10,]  0.00000000  5.28440007</span></span>
 <span><span class="co">## [11,] -1.06596713  0.00000000</span></span>
 <span><span class="co">## [12,] -0.57076709  1.09019756</span></span>
 <span><span class="co">## [13,] -0.03365904  5.62341325</span></span>
 <span><span class="co">## [14,]  0.00000000  5.62341325</span></span>
 <span><span class="co">## [15,]  0.00000000  5.62341325</span></span>
 <span><span class="co">## [16,]  0.00000000  5.62341325</span></span>
 <span><span class="co">## [17,]  0.00000000  4.70398738</span></span>
 <span><span class="co">## [18,]  0.00000000  5.62341325</span></span>
 <span><span class="co">## [19,]  0.00000000  4.97981371</span></span>
 <span><span class="co">## [20,] -0.77497987  0.00000000</span></span>
 <span><span class="co">##  [ reached getOption("max.print") -- omitted 15 rows ]</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $na.action</span></span>
 <span><span class="co">## NULL</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $xlevels</span></span>
 <span><span class="co">## named list()</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $fitted</span></span>
 <span><span class="co">##      1      2      3      4      5      6      7      8      9     10     11 </span></span>
 <span><span class="co">## setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa </span></span>
 <span><span class="co">##     12     13     14     15     16     17     18     19     20     21     22 </span></span>
 <span><span class="co">## setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa </span></span>
 <span><span class="co">##     23     24     25     26     27     28     29     30     31     32     33 </span></span>
 <span><span class="co">## setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa </span></span>
 <span><span class="co">##     34     35     36     37     38     39     40 </span></span>
 <span><span class="co">## setosa setosa setosa setosa setosa setosa setosa </span></span>
 <span><span class="co">##  [ reached getOption("max.print") -- omitted 110 entries ]</span></span>
 <span><span class="co">## Levels: setosa versicolor virginica</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $decision.values</span></span>
 <span><span class="co">##     setosa/versicolor setosa/virginica versicolor/virginica</span></span>
 <span><span class="co">## 1           1.1911739        1.0908424            1.1275805</span></span>
 <span><span class="co">## 2           1.1336557        1.0619543            1.3260964</span></span>
 <span><span class="co">## 3           1.2085065        1.0698101            1.0511345</span></span>
 <span><span class="co">## 4           1.1646153        1.0505915            1.0806874</span></span>
 <span><span class="co">## 5           1.1880814        1.0950348            0.9542815</span></span>
 <span><span class="co">## 6           1.0990761        1.0984626            0.9326361</span></span>
 <span><span class="co">## 7           1.1573474        1.0343287            0.9726843</span></span>
 <span><span class="co">## 8           1.1851598        1.0815750            1.2206802</span></span>
 <span><span class="co">## 9           1.1673499        1.0406734            0.8837945</span></span>
 <span><span class="co">## 10          1.1629911        1.0560925            1.2430067</span></span>
 <span><span class="co">## 11          1.1339282        1.0803946            1.0338357</span></span>
 <span><span class="co">## 12          1.1724182        1.0641469            1.1190423</span></span>
 <span><span class="co">## 13          1.1827355        1.0667956            1.1414844</span></span>
 <span><span class="co">##  [ reached getOption("max.print") -- omitted 137 rows ]</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $terms</span></span>
 <span><span class="co">## Species ~ Sepal.Length + Sepal.Width + Petal.Length + Petal.Width</span></span>
 <span><span class="co">## attr(,"variables")</span></span>
 <span><span class="co">## list(Species, Sepal.Length, Sepal.Width, Petal.Length, Petal.Width)</span></span>
 <span><span class="co">## attr(,"factors")</span></span>
 <span><span class="co">##              Sepal.Length Sepal.Width Petal.Length Petal.Width</span></span>
 <span><span class="co">## Species                 0           0            0           0</span></span>
 <span><span class="co">## Sepal.Length            1           0            0           0</span></span>
 <span><span class="co">## Sepal.Width             0           1            0           0</span></span>
 <span><span class="co">## Petal.Length            0           0            1           0</span></span>
 <span><span class="co">## Petal.Width             0           0            0           1</span></span>
 <span><span class="co">## attr(,"term.labels")</span></span>
 <span><span class="co">## [1] "Sepal.Length" "Sepal.Width"  "Petal.Length" "Petal.Width" </span></span>
 <span><span class="co">## attr(,"order")</span></span>
 <span><span class="co">## [1] 1 1 1 1</span></span>
 <span><span class="co">## attr(,"intercept")</span></span>
 <span><span class="co">## [1] 0</span></span>
 <span><span class="co">## attr(,"response")</span></span>
 <span><span class="co">## [1] 1</span></span>
 <span><span class="co">## attr(,".Environment")</span></span>
 <span><span class="co">## &lt;environment: 0x560e910146e8&gt;</span></span>
 <span><span class="co">## attr(,"predvars")</span></span>
 <span><span class="co">## list(Species, Sepal.Length, Sepal.Width, Petal.Length, Petal.Width)</span></span>
 <span><span class="co">## attr(,"dataClasses")</span></span>
 <span><span class="co">##      Species Sepal.Length  Sepal.Width Petal.Length  Petal.Width </span></span>
 <span><span class="co">##     "factor"    "numeric"    "numeric"    "numeric"    "numeric" </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## attr(,"class")</span></span>
 <span><span class="co">## [1] "summary.svm"</span></span></code></pre>
 </div>
 </div>
 <div class="section level3">
 <h3 id="sql-queries">SQL Queries<a class="anchor" aria-label="anchor" href="#sql-queries"></a>
 </h3>
 <p>A <code>SparkDataFrame</code> can also be registered as a temporary view in Spark SQL so that one can run SQL queries over its data. The sql function enables applications to run SQL queries programmatically and returns the result as a <code>SparkDataFrame</code>.</p>
 <div class="sourceCode" id="cb72"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">people</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/read.df.html">read.df</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="fu"><a href="../reference/sparkR.conf.html">sparkR.conf</a></span><span class="op">(</span><span class="st">"spark.home"</span><span class="op">)</span>,</span>
 <span>                         <span class="st">"/examples/src/main/resources/people.json"</span><span class="op">)</span>, <span class="st">"json"</span><span class="op">)</span></span></code></pre></div>
 <p>Register this <code>SparkDataFrame</code> as a temporary view.</p>
 <div class="sourceCode" id="cb73"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/createOrReplaceTempView.html">createOrReplaceTempView</a></span><span class="op">(</span><span class="va">people</span>, <span class="st">"people"</span><span class="op">)</span></span></code></pre></div>
 <p>SQL statements can be run using the sql method.</p>
 <div class="sourceCode" id="cb74"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">teenagers</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/sql.html">sql</a></span><span class="op">(</span><span class="st">"SELECT name FROM people WHERE age &gt;= 13 AND age &lt;= 19"</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">teenagers</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##     name</span></span>
 <span><span class="co">## 1 Justin</span></span></code></pre>
 </div>
 </div>
 <div class="section level2">
 <h2 id="machine-learning">Machine Learning<a class="anchor" aria-label="anchor" href="#machine-learning"></a>
 </h2>
 <p>SparkR supports the following machine learning models and algorithms.</p>
 <div class="section level4">
 <h4 id="classification">Classification<a class="anchor" aria-label="anchor" href="#classification"></a>
 </h4>
 <ul>
 <li><p>Linear Support Vector Machine (SVM) Classifier</p></li>
 <li><p>Logistic Regression</p></li>
 <li><p>Multilayer Perceptron (MLP)</p></li>
 <li><p>Naive Bayes</p></li>
 <li><p>Factorization Machines (FM) Classifier</p></li>
 </ul>
 </div>
 <div class="section level4">
 <h4 id="regression">Regression<a class="anchor" aria-label="anchor" href="#regression"></a>
 </h4>
 <ul>
 <li><p>Accelerated Failure Time (AFT) Survival Model</p></li>
 <li><p>Generalized Linear Model (GLM)</p></li>
 <li><p>Isotonic Regression</p></li>
 <li><p>Linear Regression</p></li>
 <li><p>Factorization Machines (FM) Regressor</p></li>
 </ul>
 </div>
 <div class="section level4">
 <h4 id="tree---classification-and-regression">Tree - Classification and Regression<a class="anchor" aria-label="anchor" href="#tree---classification-and-regression"></a>
 </h4>
 <ul>
 <li><p>Decision Tree</p></li>
 <li><p>Gradient-Boosted Trees (GBT)</p></li>
 <li><p>Random Forest</p></li>
 </ul>
 </div>
 <div class="section level4">
 <h4 id="clustering">Clustering<a class="anchor" aria-label="anchor" href="#clustering"></a>
 </h4>
 <ul>
 <li><p>Bisecting <span class="math inline">\(k\)</span>-means</p></li>
 <li><p>Gaussian Mixture Model (GMM)</p></li>
 <li><p><span class="math inline">\(k\)</span>-means Clustering</p></li>
 <li><p>Latent Dirichlet Allocation (LDA)</p></li>
 <li><p>Power Iteration Clustering (PIC)</p></li>
 </ul>
 </div>
 <div class="section level4">
 <h4 id="collaborative-filtering">Collaborative Filtering<a class="anchor" aria-label="anchor" href="#collaborative-filtering"></a>
 </h4>
 <ul>
 <li>Alternating Least Squares (ALS)</li>
 </ul>
 </div>
 <div class="section level4">
 <h4 id="frequent-pattern-mining">Frequent Pattern Mining<a class="anchor" aria-label="anchor" href="#frequent-pattern-mining"></a>
 </h4>
 <ul>
 <li>FP-growth</li>
 <li>PrefixSpan</li>
 </ul>
 </div>
 <div class="section level4">
 <h4 id="statistics">Statistics<a class="anchor" aria-label="anchor" href="#statistics"></a>
 </h4>
 <ul>
 <li>Kolmogorov-Smirnov Test</li>
 </ul>
 </div>
 <div class="section level3">
 <h3 id="r-formula">R Formula<a class="anchor" aria-label="anchor" href="#r-formula"></a>
 </h3>
 <p>For most above, SparkR supports <strong>R formula operators</strong>, including <code>~</code>, <code>.</code>, <code>:</code>, <code>+</code> and <code>-</code> for model fitting. This makes it a similar experience as using R functions.</p>
 </div>
 <div class="section level3">
 <h3 id="training-and-test-sets">Training and Test Sets<a class="anchor" aria-label="anchor" href="#training-and-test-sets"></a>
 </h3>
 <p>We can easily split <code>SparkDataFrame</code> into random training and test sets by the <code>randomSplit</code> function. It returns a list of split <code>SparkDataFrames</code> with provided <code>weights</code>. We use <code>carsDF</code> as an example and want to have about <span class="math inline">\(70%\)</span> training data and <span class="math inline">\(30%\)</span> test data.</p>
 <div class="sourceCode" id="cb76"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">splitDF_list</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/randomSplit.html">randomSplit</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0.7</span>, <span class="fl">0.3</span><span class="op">)</span>, seed <span class="op">=</span> <span class="fl">0</span><span class="op">)</span></span>
 <span><span class="va">carsDF_train</span> <span class="op">&lt;-</span> <span class="va">splitDF_list</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
 <span><span class="va">carsDF_test</span> <span class="op">&lt;-</span> <span class="va">splitDF_list</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span></code></pre></div>
 <div class="sourceCode" id="cb77"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/count.html">count</a></span><span class="op">(</span><span class="va">carsDF_train</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] 24</span></span></code></pre>
 <div class="sourceCode" id="cb79"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">carsDF_train</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##                model  mpg cyl disp  hp drat    wt  qsec vs am gear carb</span></span>
 <span><span class="co">## 1 Cadillac Fleetwood 10.4   8  472 205 2.93 5.250 17.98  0  0    3    4</span></span>
 <span><span class="co">## 2         Camaro Z28 13.3   8  350 245 3.73 3.840 15.41  0  0    3    4</span></span>
 <span><span class="co">## 3  Chrysler Imperial 14.7   8  440 230 3.23 5.345 17.42  0  0    3    4</span></span>
 <span><span class="co">## 4   Dodge Challenger 15.5   8  318 150 2.76 3.520 16.87  0  0    3    2</span></span>
 <span><span class="co">## 5         Duster 360 14.3   8  360 245 3.21 3.570 15.84  0  0    3    4</span></span>
 <span><span class="co">## 6       Ferrari Dino 19.7   6  145 175 3.62 2.770 15.50  0  1    5    6</span></span></code></pre>
 <div class="sourceCode" id="cb81"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/count.html">count</a></span><span class="op">(</span><span class="va">carsDF_test</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] 8</span></span></code></pre>
 <div class="sourceCode" id="cb83"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">carsDF_test</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##            model  mpg cyl  disp  hp drat    wt  qsec vs am gear carb</span></span>
 <span><span class="co">## 1    AMC Javelin 15.2   8 304.0 150 3.15 3.435 17.30  0  0    3    2</span></span>
 <span><span class="co">## 2     Datsun 710 22.8   4 108.0  93 3.85 2.320 18.61  1  1    4    1</span></span>
 <span><span class="co">## 3       Fiat 128 32.4   4  78.7  66 4.08 2.200 19.47  1  1    4    1</span></span>
 <span><span class="co">## 4      Merc 240D 24.4   4 146.7  62 3.69 3.190 20.00  1  0    4    2</span></span>
 <span><span class="co">## 5       Merc 280 19.2   6 167.6 123 3.92 3.440 18.30  1  0    4    4</span></span>
 <span><span class="co">## 6 Toyota Corolla 33.9   4  71.1  65 4.22 1.835 19.90  1  1    4    1</span></span></code></pre>
 </div>
 <div class="section level3">
 <h3 id="models-and-algorithms">Models and Algorithms<a class="anchor" aria-label="anchor" href="#models-and-algorithms"></a>
 </h3>
 <div class="section level4">
 <h4 id="linear-support-vector-machine-svm-classifier">Linear Support Vector Machine (SVM) Classifier<a class="anchor" aria-label="anchor" href="#linear-support-vector-machine-svm-classifier"></a>
 </h4>
 <p><a href="https://en.wikipedia.org/wiki/Support_vector_machine#Linear_SVM" class="external-link">Linear Support Vector Machine (SVM)</a> classifier is an SVM classifier with linear kernels. This is a binary classifier. We use a simple example to show how to use <code>spark.svmLinear</code> for binary classification.</p>
 <div class="sourceCode" id="cb85"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="co"># load training data and create a DataFrame</span></span>
 <span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">training</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="co"># fit a Linear SVM classifier model</span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.svmLinear.html">spark.svmLinear</a></span><span class="op">(</span><span class="va">training</span>,  <span class="va">Survived</span> <span class="op">~</span> <span class="va">.</span>, regParam <span class="op">=</span> <span class="fl">0.01</span>, maxIter <span class="op">=</span> <span class="fl">10</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $coefficients</span></span>
 <span><span class="co">##                 Estimate</span></span>
 <span><span class="co">## (Intercept)  0.993131388</span></span>
 <span><span class="co">## Class_1st   -0.386500359</span></span>
 <span><span class="co">## Class_2nd   -0.622627816</span></span>
 <span><span class="co">## Class_3rd   -0.204446602</span></span>
 <span><span class="co">## Sex_Female  -0.589950309</span></span>
 <span><span class="co">## Age_Adult    0.741676902</span></span>
 <span><span class="co">## Freq        -0.006582887</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $numClasses</span></span>
 <span><span class="co">## [1] 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $numFeatures</span></span>
 <span><span class="co">## [1] 6</span></span></code></pre>
 <p>Predict values on training data</p>
 <div class="sourceCode" id="cb87"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">prediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">training</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">prediction</span>, <span class="st">"Class"</span>, <span class="st">"Sex"</span>, <span class="st">"Age"</span>, <span class="st">"Freq"</span>, <span class="st">"Survived"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class    Sex   Age Freq Survived prediction</span></span>
 <span><span class="co">## 1   1st   Male Child    0       No        Yes</span></span>
 <span><span class="co">## 2   2nd   Male Child    0       No        Yes</span></span>
 <span><span class="co">## 3   3rd   Male Child   35       No        Yes</span></span>
 <span><span class="co">## 4  Crew   Male Child    0       No        Yes</span></span>
 <span><span class="co">## 5   1st Female Child    0       No        Yes</span></span>
 <span><span class="co">## 6   2nd Female Child    0       No         No</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="logistic-regression">Logistic Regression<a class="anchor" aria-label="anchor" href="#logistic-regression"></a>
 </h4>
 <p><a href="https://en.wikipedia.org/wiki/Logistic_regression" class="external-link">Logistic regression</a> is a widely-used model when the response is categorical. It can be seen as a special case of the <a href="https://en.wikipedia.org/wiki/Generalized_linear_model" class="external-link">Generalized Linear Predictive Model</a>. We provide <code>spark.logit</code> on top of <code>spark.glm</code> to support logistic regression with advanced hyper-parameters. It supports both binary and multiclass classification with elastic-net regularization and feature standardization, similar to <code>glmnet</code>.</p>
 <p>We use a simple example to demonstrate <code>spark.logit</code> usage. In general, there are three steps of using <code>spark.logit</code>: 1). Create a dataframe from a proper data source; 2). Fit a logistic regression model using <code>spark.logit</code> with a proper parameter setting; and 3). Obtain the coefficient matrix of the fitted model using <code>summary</code> and use the model for prediction with <code>predict</code>.</p>
 <p>Binomial logistic regression</p>
 <div class="sourceCode" id="cb89"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">training</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.logit.html">spark.logit</a></span><span class="op">(</span><span class="va">training</span>, <span class="va">Survived</span> <span class="op">~</span> <span class="va">.</span>, regParam <span class="op">=</span> <span class="fl">0.04741301</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $coefficients</span></span>
 <span><span class="co">##                  Estimate</span></span>
 <span><span class="co">## (Intercept)  0.2255014282</span></span>
 <span><span class="co">## Class_1st   -0.1338856652</span></span>
 <span><span class="co">## Class_2nd   -0.1479826947</span></span>
 <span><span class="co">## Class_3rd    0.0005674937</span></span>
 <span><span class="co">## Sex_Female  -0.2011183871</span></span>
 <span><span class="co">## Age_Adult    0.3263186885</span></span>
 <span><span class="co">## Freq        -0.0033111157</span></span></code></pre>
 <p>Predict values on training data</p>
 <div class="sourceCode" id="cb91"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">fitted</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">training</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">fitted</span>, <span class="st">"Class"</span>, <span class="st">"Sex"</span>, <span class="st">"Age"</span>, <span class="st">"Freq"</span>, <span class="st">"Survived"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class    Sex   Age Freq Survived prediction</span></span>
 <span><span class="co">## 1   1st   Male Child    0       No        Yes</span></span>
 <span><span class="co">## 2   2nd   Male Child    0       No        Yes</span></span>
 <span><span class="co">## 3   3rd   Male Child   35       No        Yes</span></span>
 <span><span class="co">## 4  Crew   Male Child    0       No        Yes</span></span>
 <span><span class="co">## 5   1st Female Child    0       No         No</span></span>
 <span><span class="co">## 6   2nd Female Child    0       No         No</span></span></code></pre>
 <p>Multinomial logistic regression against three classes</p>
 <div class="sourceCode" id="cb93"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">training</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="co"># Note in this case, Spark infers it is multinomial logistic regression, so family = "multinomial" is optional.</span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.logit.html">spark.logit</a></span><span class="op">(</span><span class="va">training</span>, <span class="va">Class</span> <span class="op">~</span> <span class="va">.</span>, regParam <span class="op">=</span> <span class="fl">0.07815179</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $coefficients</span></span>
 <span><span class="co">##                      1st          2nd          3rd         Crew</span></span>
 <span><span class="co">## (Intercept)  0.051662845  0.062998145 -0.039083689 -0.075577300</span></span>
 <span><span class="co">## Sex_Female  -0.088030587 -0.102528148  0.059233106  0.131325629</span></span>
 <span><span class="co">## Age_Adult    0.141935316  0.169492058 -0.102562719 -0.208864654</span></span>
 <span><span class="co">## Survived_No  0.052721020  0.057980057 -0.029408423 -0.081292653</span></span>
 <span><span class="co">## Freq        -0.001555912 -0.001970377  0.001303836  0.002222453</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="multilayer-perceptron">Multilayer Perceptron<a class="anchor" aria-label="anchor" href="#multilayer-perceptron"></a>
 </h4>
 <p>Multilayer perceptron classifier (MLPC) is a classifier based on the <a href="https://en.wikipedia.org/wiki/Feedforward_neural_network" class="external-link">feedforward artificial neural network</a>. MLPC consists of multiple layers of nodes. Each layer is fully connected to the next layer in the network. Nodes in the input layer represent the input data. All other nodes map inputs to outputs by a linear combination of the inputs with the node’s weights <span class="math inline">\(w\)</span> and bias <span class="math inline">\(b\)</span> and applying an activation function. This can be written in matrix form for MLPC with <span class="math inline">\(K+1\)</span> layers as follows: <span class="math display">\[
 y(x)=f_K(\ldots f_2(w_2^T f_1(w_1^T x + b_1) + b_2) \ldots + b_K).
 \]</span></p>
 <p>Nodes in intermediate layers use sigmoid (logistic) function: <span class="math display">\[
 f(z_i) = \frac{1}{1+e^{-z_i}}.
 \]</span></p>
 <p>Nodes in the output layer use softmax function: <span class="math display">\[
 f(z_i) = \frac{e^{z_i}}{\sum_{k=1}^N e^{z_k}}.
 \]</span></p>
 <p>The number of nodes <span class="math inline">\(N\)</span> in the output layer corresponds to the number of classes.</p>
 <p>MLPC employs backpropagation for learning the model. We use the logistic loss function for optimization and L-BFGS as an optimization routine.</p>
 <p><code>spark.mlp</code> requires at least two columns in <code>data</code>: one named <code>"label"</code> and the other one <code>"features"</code>. The <code>"features"</code> column should be in libSVM-format.</p>
 <p>We use Titanic data set to show how to use <code>spark.mlp</code> in classification.</p>
 <div class="sourceCode" id="cb95"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">training</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="co"># fit a Multilayer Perceptron Classification Model</span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.mlp.html">spark.mlp</a></span><span class="op">(</span><span class="va">training</span>, <span class="va">Survived</span> <span class="op">~</span> <span class="va">Age</span> <span class="op">+</span> <span class="va">Sex</span>, blockSize <span class="op">=</span> <span class="fl">128</span>, layers <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">2</span>, <span class="fl">2</span><span class="op">)</span>, solver <span class="op">=</span> <span class="st">"l-bfgs"</span>, maxIter <span class="op">=</span> <span class="fl">100</span>, tol <span class="op">=</span> <span class="fl">0.5</span>, stepSize <span class="op">=</span> <span class="fl">1</span>, seed <span class="op">=</span> <span class="fl">1</span>, initialWeights <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span> <span class="fl">0</span>, <span class="fl">0</span>, <span class="fl">5</span>, <span class="fl">5</span>, <span class="fl">9</span>, <span class="fl">9</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <p>To avoid lengthy display, we only present partial results of the model summary. You can check the full result from your sparkR shell.</p>
 <div class="sourceCode" id="cb96"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="co"># check the summary of the fitted model</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $numOfInputs</span></span>
 <span><span class="co">## [1] 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $numOfOutputs</span></span>
 <span><span class="co">## [1] 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $layers</span></span>
 <span><span class="co">## [1] 2 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $weights</span></span>
 <span><span class="co">## $weights[[1]]</span></span>
 <span><span class="co">## [1] 0</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $weights[[2]]</span></span>
 <span><span class="co">## [1] 0</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $weights[[3]]</span></span>
 <span><span class="co">## [1] 5</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $weights[[4]]</span></span>
 <span><span class="co">## [1] 5</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $weights[[5]]</span></span>
 <span><span class="co">## [1] 9</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $weights[[6]]</span></span>
 <span><span class="co">## [1] 9</span></span></code></pre>
 <div class="sourceCode" id="cb98"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="co"># make predictions use the fitted model</span></span>
 <span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">training</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">predictions</span>, <span class="va">predictions</span><span class="op">$</span><span class="va">prediction</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   prediction</span></span>
 <span><span class="co">## 1         No</span></span>
 <span><span class="co">## 2         No</span></span>
 <span><span class="co">## 3         No</span></span>
 <span><span class="co">## 4         No</span></span>
 <span><span class="co">## 5         No</span></span>
 <span><span class="co">## 6         No</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="naive-bayes">Naive Bayes<a class="anchor" aria-label="anchor" href="#naive-bayes"></a>
 </h4>
 <p>Naive Bayes model assumes independence among the features. <code>spark.naiveBayes</code> fits a <a href="https://en.wikipedia.org/wiki/Naive_Bayes_classifier#Bernoulli_naive_Bayes" class="external-link">Bernoulli naive Bayes model</a> against a SparkDataFrame. The data should be all categorical. These models are often used for document classification.</p>
 <div class="sourceCode" id="cb100"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">titanic</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">titanicDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">titanic</span><span class="op">[</span><span class="va">titanic</span><span class="op">$</span><span class="va">Freq</span> <span class="op">&gt;</span> <span class="fl">0</span>, <span class="op">-</span><span class="fl">5</span><span class="op">]</span><span class="op">)</span></span>
 <span><span class="va">naiveBayesModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.naiveBayes.html">spark.naiveBayes</a></span><span class="op">(</span><span class="va">titanicDF</span>, <span class="va">Survived</span> <span class="op">~</span> <span class="va">Class</span> <span class="op">+</span> <span class="va">Sex</span> <span class="op">+</span> <span class="va">Age</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">naiveBayesModel</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $apriori</span></span>
 <span><span class="co">##            Yes        No</span></span>
 <span><span class="co">## [1,] 0.5769231 0.4230769</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $tables</span></span>
 <span><span class="co">##     Class_3rd Class_1st Class_2nd Sex_Female Age_Adult</span></span>
 <span><span class="co">## Yes 0.3125    0.3125    0.3125    0.5        0.5625   </span></span>
 <span><span class="co">## No  0.4166667 0.25      0.25      0.5        0.75</span></span></code></pre>
 <div class="sourceCode" id="cb102"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">naiveBayesPrediction</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">naiveBayesModel</span>, <span class="va">titanicDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">naiveBayesPrediction</span>, <span class="st">"Class"</span>, <span class="st">"Sex"</span>, <span class="st">"Age"</span>, <span class="st">"Survived"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class    Sex   Age Survived prediction</span></span>
 <span><span class="co">## 1   3rd   Male Child       No        Yes</span></span>
 <span><span class="co">## 2   3rd Female Child       No        Yes</span></span>
 <span><span class="co">## 3   1st   Male Adult       No        Yes</span></span>
 <span><span class="co">## 4   2nd   Male Adult       No        Yes</span></span>
 <span><span class="co">## 5   3rd   Male Adult       No         No</span></span>
 <span><span class="co">## 6  Crew   Male Adult       No        Yes</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="factorization-machines-classifier">Factorization Machines Classifier<a class="anchor" aria-label="anchor" href="#factorization-machines-classifier"></a>
 </h4>
 <p>Factorization Machines for classification problems.</p>
 <p>For background and details about the implementation of factorization machines, refer to the <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#factorization-machines" class="external-link">Factorization Machines section</a>.</p>
 <div class="sourceCode" id="cb104"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">training</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.fmClassifier.html">spark.fmClassifier</a></span><span class="op">(</span><span class="va">training</span>, <span class="va">Survived</span> <span class="op">~</span> <span class="va">Age</span> <span class="op">+</span> <span class="va">Sex</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $coefficients</span></span>
 <span><span class="co">##                 Estimate</span></span>
 <span><span class="co">## (Intercept) 0.0064275991</span></span>
 <span><span class="co">## Age_Adult   0.0001294448</span></span>
 <span><span class="co">## Sex_Female  0.0001294448</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $factors</span></span>
 <span><span class="co">##            [,1]        [,2]      [,3]       [,4]        [,5]        [,6]</span></span>
 <span><span class="co">## [1,] -0.3256224  0.11912568 0.1460235  0.1620567  0.13153516  0.06403695</span></span>
 <span><span class="co">## [2,] -0.1382155 -0.03658261 0.1717808 -0.1602241 -0.08446129 -0.19287098</span></span>
 <span><span class="co">##             [,7]        [,8]</span></span>
 <span><span class="co">## [1,] -0.03292446 -0.05166818</span></span>
 <span><span class="co">## [2,]  0.19252571  0.06237194</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $numClasses</span></span>
 <span><span class="co">## [1] 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $numFeatures</span></span>
 <span><span class="co">## [1] 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $factorSize</span></span>
 <span><span class="co">## [1] 8</span></span></code></pre>
 <div class="sourceCode" id="cb106"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">training</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">predictions</span>, <span class="va">predictions</span><span class="op">$</span><span class="va">prediction</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   prediction</span></span>
 <span><span class="co">## 1        Yes</span></span>
 <span><span class="co">## 2        Yes</span></span>
 <span><span class="co">## 3        Yes</span></span>
 <span><span class="co">## 4        Yes</span></span>
 <span><span class="co">## 5        Yes</span></span>
 <span><span class="co">## 6        Yes</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="accelerated-failure-time-survival-model">Accelerated Failure Time Survival Model<a class="anchor" aria-label="anchor" href="#accelerated-failure-time-survival-model"></a>
 </h4>
 <p>Survival analysis studies the expected duration of time until an event happens, and often the relationship with risk factors or treatment taken on the subject. In contrast to standard regression analysis, survival modeling has to deal with special characteristics in the data including non-negative survival time and censoring.</p>
 <p>Accelerated Failure Time (AFT) model is a parametric survival model for censored data that assumes the effect of a covariate is to accelerate or decelerate the life course of an event by some constant. For more information, refer to the Wikipedia page <a href="https://en.wikipedia.org/wiki/Accelerated_failure_time_model" class="external-link">AFT Model</a> and the references there. Different from a <a href="https://en.wikipedia.org/wiki/Proportional_hazards_model" class="external-link">Proportional Hazards Model</a> designed for the same purpose, the AFT model is easier to parallelize because each instance contributes to the objective function independently.</p>
 <div class="sourceCode" id="cb108"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="kw"><a href="https://rdrr.io/r/base/library.html" class="external-link">library</a></span><span class="op">(</span><span class="va"><a href="https://github.com/therneau/survival" class="external-link">survival</a></span><span class="op">)</span></span>
 <span><span class="va">ovarianDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">ovarian</span><span class="op">)</span></span>
 <span><span class="va">aftModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.survreg.html">spark.survreg</a></span><span class="op">(</span><span class="va">ovarianDF</span>, <span class="fu"><a href="https://rdrr.io/pkg/survival/man/Surv.html" class="external-link">Surv</a></span><span class="op">(</span><span class="va">futime</span>, <span class="va">fustat</span><span class="op">)</span> <span class="op">~</span> <span class="va">ecog_ps</span> <span class="op">+</span> <span class="va">rx</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">aftModel</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $coefficients</span></span>
 <span><span class="co">##                  Value</span></span>
 <span><span class="co">## (Intercept)  6.8966910</span></span>
 <span><span class="co">## ecog_ps     -0.3850414</span></span>
 <span><span class="co">## rx           0.5286455</span></span>
 <span><span class="co">## Log(scale)  -0.1234429</span></span></code></pre>
 <div class="sourceCode" id="cb110"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">aftPredictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">aftModel</span>, <span class="va">ovarianDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">aftPredictions</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   futime fustat     age resid_ds rx ecog_ps label prediction</span></span>
 <span><span class="co">## 1     59      1 72.3315        2  1       1    59   1141.724</span></span>
 <span><span class="co">## 2    115      1 74.4932        2  1       1   115   1141.724</span></span>
 <span><span class="co">## 3    156      1 66.4658        2  1       2   156    776.855</span></span>
 <span><span class="co">## 4    421      0 53.3644        2  2       1   421   1937.087</span></span>
 <span><span class="co">## 5    431      1 50.3397        2  1       1   431   1141.724</span></span>
 <span><span class="co">## 6    448      0 56.4301        1  1       2   448    776.855</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="generalized-linear-model">Generalized Linear Model<a class="anchor" aria-label="anchor" href="#generalized-linear-model"></a>
 </h4>
 <p>The main function is <code>spark.glm</code>. The following families and link functions are supported. The default is gaussian.</p>
 <table class="table">
 <thead><tr class="header">
 <th>Family</th>
 <th>Link Function</th>
 </tr></thead>
 <tbody>
 <tr class="odd">
 <td>gaussian</td>
 <td>identity, log, inverse</td>
 </tr>
 <tr class="even">
 <td>binomial</td>
 <td>logit, probit, cloglog (complementary log-log)</td>
 </tr>
 <tr class="odd">
 <td>poisson</td>
 <td>log, identity, sqrt</td>
 </tr>
 <tr class="even">
 <td>gamma</td>
 <td>inverse, identity, log</td>
 </tr>
 <tr class="odd">
 <td>tweedie</td>
 <td>power link function</td>
 </tr>
 </tbody>
 </table>
 <p>There are three ways to specify the <code>family</code> argument.</p>
 <ul>
 <li><p>Family name as a character string, e.g. <code>family = "gaussian"</code>.</p></li>
 <li><p>Family function, e.g. <code>family = binomial</code>.</p></li>
 <li><p>Result returned by a family function, e.g. <code>family = poisson(link = log)</code>.</p></li>
 <li>
 <p>Note that there are two ways to specify the tweedie family:</p>
 <ol style="list-style-type: lower-alpha">
 <li>Set <code>family = "tweedie"</code> and specify the <code>var.power</code> and <code>link.power</code>
 </li>
 <li>When package <code>statmod</code> is loaded, the tweedie family is specified using the family definition therein, i.e., <code><a href="https://rdrr.io/pkg/statmod/man/tweedie.html" class="external-link">tweedie()</a></code>.</li>
 </ol>
 </li>
 </ul>
 <p>For more information regarding the families and their link functions, see the Wikipedia page <a href="https://en.wikipedia.org/wiki/Generalized_linear_model" class="external-link">Generalized Linear Model</a>.</p>
 <p>We use the <code>mtcars</code> dataset as an illustration. The corresponding <code>SparkDataFrame</code> is <code>carsDF</code>. After fitting the model, we print out a summary and see the fitted values by making predictions on the original dataset. We can also pass into a new <code>SparkDataFrame</code> of same schema to predict on new data.</p>
 <div class="sourceCode" id="cb112"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">gaussianGLM</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.glm.html">spark.glm</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">mpg</span> <span class="op">~</span> <span class="va">wt</span> <span class="op">+</span> <span class="va">hp</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">gaussianGLM</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## </span></span>
 <span><span class="co">## Deviance Residuals: </span></span>
 <span><span class="co">## (Note: These are approximate quantiles with relative error &lt;= 0.01)</span></span>
 <span><span class="co">##     Min       1Q   Median       3Q      Max  </span></span>
 <span><span class="co">## -3.9410  -1.6499  -0.3267   1.0373   5.8538  </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Coefficients:</span></span>
 <span><span class="co">##               Estimate  Std. Error  t value    Pr(&gt;|t|)</span></span>
 <span><span class="co">## (Intercept)  37.227270   1.5987875  23.2847  0.0000e+00</span></span>
 <span><span class="co">## wt           -3.877831   0.6327335  -6.1287  1.1196e-06</span></span>
 <span><span class="co">## hp           -0.031773   0.0090297  -3.5187  1.4512e-03</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## (Dispersion parameter for gaussian family taken to be 6.725785)</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">##     Null deviance: 1126.05  on 31  degrees of freedom</span></span>
 <span><span class="co">## Residual deviance:  195.05  on 29  degrees of freedom</span></span>
 <span><span class="co">## AIC: 156.7</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Number of Fisher Scoring iterations: 1</span></span></code></pre>
 <p>When doing prediction, a new column called <code>prediction</code> will be appended. Let’s look at only a subset of columns here.</p>
 <div class="sourceCode" id="cb114"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">gaussianFitted</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">gaussianGLM</span>, <span class="va">carsDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">gaussianFitted</span>, <span class="st">"model"</span>, <span class="st">"prediction"</span>, <span class="st">"mpg"</span>, <span class="st">"wt"</span>, <span class="st">"hp"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##               model prediction  mpg    wt  hp</span></span>
 <span><span class="co">## 1         Mazda RX4   23.57233 21.0 2.620 110</span></span>
 <span><span class="co">## 2     Mazda RX4 Wag   22.58348 21.0 2.875 110</span></span>
 <span><span class="co">## 3        Datsun 710   25.27582 22.8 2.320  93</span></span>
 <span><span class="co">## 4    Hornet 4 Drive   21.26502 21.4 3.215 110</span></span>
 <span><span class="co">## 5 Hornet Sportabout   18.32727 18.7 3.440 175</span></span>
 <span><span class="co">## 6           Valiant   20.47382 18.1 3.460 105</span></span></code></pre>
 <p>The following is the same fit using the tweedie family:</p>
 <div class="sourceCode" id="cb116"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">tweedieGLM1</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.glm.html">spark.glm</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">mpg</span> <span class="op">~</span> <span class="va">wt</span> <span class="op">+</span> <span class="va">hp</span>, family <span class="op">=</span> <span class="st">"tweedie"</span>, var.power <span class="op">=</span> <span class="fl">0.0</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">tweedieGLM1</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## </span></span>
 <span><span class="co">## Deviance Residuals: </span></span>
 <span><span class="co">## (Note: These are approximate quantiles with relative error &lt;= 0.01)</span></span>
 <span><span class="co">##     Min       1Q   Median       3Q      Max  </span></span>
 <span><span class="co">## -3.9410  -1.6499  -0.3267   1.0373   5.8538  </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Coefficients:</span></span>
 <span><span class="co">##               Estimate  Std. Error  t value    Pr(&gt;|t|)</span></span>
 <span><span class="co">## (Intercept)  37.227270   1.5987875  23.2847  0.0000e+00</span></span>
 <span><span class="co">## wt           -3.877831   0.6327335  -6.1287  1.1196e-06</span></span>
 <span><span class="co">## hp           -0.031773   0.0090297  -3.5187  1.4512e-03</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## (Dispersion parameter for tweedie family taken to be 6.725785)</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">##     Null deviance: 1126.05  on 31  degrees of freedom</span></span>
 <span><span class="co">## Residual deviance:  195.05  on 29  degrees of freedom</span></span>
 <span><span class="co">## AIC: 156.7</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Number of Fisher Scoring iterations: 1</span></span></code></pre>
 <p>We can try other distributions in the tweedie family, for example, a compound Poisson distribution with a log link:</p>
 <div class="sourceCode" id="cb118"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">tweedieGLM2</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.glm.html">spark.glm</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">mpg</span> <span class="op">~</span> <span class="va">wt</span> <span class="op">+</span> <span class="va">hp</span>, family <span class="op">=</span> <span class="st">"tweedie"</span>,</span>
 <span>                         var.power <span class="op">=</span> <span class="fl">1.2</span>, link.power <span class="op">=</span> <span class="fl">0.0</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">tweedieGLM2</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## </span></span>
 <span><span class="co">## Deviance Residuals: </span></span>
 <span><span class="co">## (Note: These are approximate quantiles with relative error &lt;= 0.01)</span></span>
 <span><span class="co">##      Min        1Q    Median        3Q       Max  </span></span>
 <span><span class="co">## -0.58074  -0.25335  -0.09892   0.18608   0.82717  </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Coefficients:</span></span>
 <span><span class="co">##                Estimate  Std. Error  t value    Pr(&gt;|t|)</span></span>
 <span><span class="co">## (Intercept)   3.8500849  0.06698272  57.4788  0.0000e+00</span></span>
 <span><span class="co">## wt           -0.2018426  0.02897283  -6.9666  1.1691e-07</span></span>
 <span><span class="co">## hp           -0.0016248  0.00041603  -3.9054  5.1697e-04</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## (Dispersion parameter for tweedie family taken to be 0.1340111)</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">##     Null deviance: 29.8820  on 31  degrees of freedom</span></span>
 <span><span class="co">## Residual deviance:  3.7739  on 29  degrees of freedom</span></span>
 <span><span class="co">## AIC: NA</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Number of Fisher Scoring iterations: 4</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="isotonic-regression">Isotonic Regression<a class="anchor" aria-label="anchor" href="#isotonic-regression"></a>
 </h4>
 <p><code>spark.isoreg</code> fits an <a href="https://en.wikipedia.org/wiki/Isotonic_regression" class="external-link">Isotonic Regression</a> model against a <code>SparkDataFrame</code>. It solves a weighted univariate a regression problem under a complete order constraint. Specifically, given a set of real observed responses <span class="math inline">\(y_1, \ldots, y_n\)</span>, corresponding real features <span class="math inline">\(x_1, \ldots, x_n\)</span>, and optionally positive weights <span class="math inline">\(w_1, \ldots, w_n\)</span>, we want to find a monotone (piecewise linear) function <span class="math inline">\(f\)</span> to minimize <span class="math display">\[
 \ell(f) = \sum_{i=1}^n w_i (y_i - f(x_i))^2.
 \]</span></p>
 <p>There are a few more arguments that may be useful.</p>
 <ul>
 <li><p><code>weightCol</code>: a character string specifying the weight column.</p></li>
 <li><p><code>isotonic</code>: logical value indicating whether the output sequence should be isotonic/increasing (<code>TRUE</code>) or antitonic/decreasing (<code>FALSE</code>).</p></li>
 <li><p><code>featureIndex</code>: the index of the feature on the right hand side of the formula if it is a vector column (default: 0), no effect otherwise.</p></li>
 </ul>
 <p>We use an artificial example to show the use.</p>
 <div class="sourceCode" id="cb120"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">y</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">3.0</span>, <span class="fl">6.0</span>, <span class="fl">8.0</span>, <span class="fl">5.0</span>, <span class="fl">7.0</span><span class="op">)</span></span>
 <span><span class="va">x</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1.0</span>, <span class="fl">2.0</span>, <span class="fl">3.5</span>, <span class="fl">3.0</span>, <span class="fl">4.0</span><span class="op">)</span></span>
 <span><span class="va">w</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/rep.html" class="external-link">rep</a></span><span class="op">(</span><span class="fl">1.0</span>, <span class="fl">5</span><span class="op">)</span></span>
 <span><span class="va">data</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>y <span class="op">=</span> <span class="va">y</span>, x <span class="op">=</span> <span class="va">x</span>, w <span class="op">=</span> <span class="va">w</span><span class="op">)</span></span>
 <span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">data</span><span class="op">)</span></span>
 <span><span class="va">isoregModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.isoreg.html">spark.isoreg</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">y</span> <span class="op">~</span> <span class="va">x</span>, weightCol <span class="op">=</span> <span class="st">"w"</span><span class="op">)</span></span>
 <span><span class="va">isoregFitted</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">isoregModel</span>, <span class="va">df</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">isoregFitted</span>, <span class="st">"x"</span>, <span class="st">"y"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##     x y prediction</span></span>
 <span><span class="co">## 1 1.0 3        3.0</span></span>
 <span><span class="co">## 2 2.0 6        5.5</span></span>
 <span><span class="co">## 3 3.5 8        7.5</span></span>
 <span><span class="co">## 4 3.0 5        5.5</span></span>
 <span><span class="co">## 5 4.0 7        7.5</span></span></code></pre>
 <p>In the prediction stage, based on the fitted monotone piecewise function, the rules are:</p>
 <ul>
 <li><p>If the prediction input exactly matches a training feature then associated prediction is returned. In case there are multiple predictions with the same feature then one of them is returned. Which one is undefined.</p></li>
 <li><p>If the prediction input is lower or higher than all training features then prediction with lowest or highest feature is returned respectively. In case there are multiple predictions with the same feature then the lowest or highest is returned respectively.</p></li>
 <li><p>If the prediction input falls between two training features then prediction is treated as piecewise linear function and interpolated value is calculated from the predictions of the two closest features. In case there are multiple values with the same feature then the same rules as in previous point are used.</p></li>
 </ul>
 <p>For example, when the input is <span class="math inline">\(3.2\)</span>, the two closest feature values are <span class="math inline">\(3.0\)</span> and <span class="math inline">\(3.5\)</span>, then predicted value would be a linear interpolation between the predicted values at <span class="math inline">\(3.0\)</span> and <span class="math inline">\(3.5\)</span>.</p>
 <div class="sourceCode" id="cb122"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">newDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>x <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1.5</span>, <span class="fl">3.2</span><span class="op">)</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">isoregModel</span>, <span class="va">newDF</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##     x prediction</span></span>
 <span><span class="co">## 1 1.5       4.25</span></span>
 <span><span class="co">## 2 3.2       6.30</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="linear-regression">Linear Regression<a class="anchor" aria-label="anchor" href="#linear-regression"></a>
 </h4>
 <p>Linear regression model.</p>
 <div class="sourceCode" id="cb124"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.lm.html">spark.lm</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">mpg</span> <span class="op">~</span> <span class="va">wt</span> <span class="op">+</span> <span class="va">hp</span><span class="op">)</span></span>
 <span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $coefficients</span></span>
 <span><span class="co">##                Estimate</span></span>
 <span><span class="co">## (Intercept) 37.22727012</span></span>
 <span><span class="co">## wt          -3.87783074</span></span>
 <span><span class="co">## hp          -0.03177295</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $numFeatures</span></span>
 <span><span class="co">## [1] 2</span></span></code></pre>
 <div class="sourceCode" id="cb126"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">carsDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">predictions</span>, <span class="va">predictions</span><span class="op">$</span><span class="va">prediction</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   prediction</span></span>
 <span><span class="co">## 1   23.57233</span></span>
 <span><span class="co">## 2   22.58348</span></span>
 <span><span class="co">## 3   25.27582</span></span>
 <span><span class="co">## 4   21.26502</span></span>
 <span><span class="co">## 5   18.32727</span></span>
 <span><span class="co">## 6   20.47382</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="factorization-machines-regressor">Factorization Machines Regressor<a class="anchor" aria-label="anchor" href="#factorization-machines-regressor"></a>
 </h4>
 <p>Factorization Machines for regression problems.</p>
 <p>For background and details about the implementation of factorization machines, refer to the <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#factorization-machines" class="external-link">Factorization Machines section</a>.</p>
 <div class="sourceCode" id="cb128"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.fmRegressor.html">spark.fmRegressor</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="va">mpg</span> <span class="op">~</span> <span class="va">wt</span> <span class="op">+</span> <span class="va">hp</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $coefficients</span></span>
 <span><span class="co">##              Estimate</span></span>
 <span><span class="co">## (Intercept) 0.1518559</span></span>
 <span><span class="co">## wt          3.6472555</span></span>
 <span><span class="co">## hp          2.8026828</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $factors</span></span>
 <span><span class="co">##            [,1]       [,2]       [,3]       [,4]      [,5]       [,6]</span></span>
 <span><span class="co">## [1,]  0.1424420 -0.1178110 -0.3970272 -0.4696695  0.400288  0.3690930</span></span>
 <span><span class="co">## [2,] -0.1626185  0.1512138  0.3690435  0.4076975 -0.625752 -0.3715109</span></span>
 <span><span class="co">##             [,7]       [,8]</span></span>
 <span><span class="co">## [1,]  0.03472468 -0.1703219</span></span>
 <span><span class="co">## [2,] -0.02109148 -0.2006249</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $numFeatures</span></span>
 <span><span class="co">## [1] 2</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $factorSize</span></span>
 <span><span class="co">## [1] 8</span></span></code></pre>
 <div class="sourceCode" id="cb130"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">carsDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">predictions</span>, <span class="va">predictions</span><span class="op">$</span><span class="va">prediction</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   prediction</span></span>
 <span><span class="co">## 1  106.70996</span></span>
 <span><span class="co">## 2   87.07526</span></span>
 <span><span class="co">## 3  111.07931</span></span>
 <span><span class="co">## 4   60.89565</span></span>
 <span><span class="co">## 5   61.81374</span></span>
 <span><span class="co">## 6   40.70095</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="decision-tree">Decision Tree<a class="anchor" aria-label="anchor" href="#decision-tree"></a>
 </h4>
 <p><code>spark.decisionTree</code> fits a <a href="https://en.wikipedia.org/wiki/Decision_tree_learning" class="external-link">decision tree</a> classification or regression model on a <code>SparkDataFrame</code>. Users can call <code>summary</code> to get a summary of the fitted model, <code>predict</code> to make predictions, and <code>write.ml</code>/<code>read.ml</code> to save/load fitted models.</p>
 <p>We use the <code>Titanic</code> dataset to train a decision tree and make predictions:</p>
 <div class="sourceCode" id="cb132"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="va">dtModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.decisionTree.html">spark.decisionTree</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">Survived</span> <span class="op">~</span> <span class="va">.</span>, type <span class="op">=</span> <span class="st">"classification"</span>, maxDepth <span class="op">=</span> <span class="fl">2</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">dtModel</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## Formula:  Survived ~ .</span></span>
 <span><span class="co">## Number of features:  6</span></span>
 <span><span class="co">## Features:  Class_1st Class_2nd Class_3rd Sex_Female Age_Adult Freq</span></span>
 <span><span class="co">## Feature importances:  (6,[5],[1.0])</span></span>
 <span><span class="co">## Max Depth:  2</span></span>
 <span><span class="co">##  DecisionTreeClassificationModel: uid=dtc_e86a0aa68eb6, depth=2, numNodes=5, numClasses=2, numFeatures=6</span></span>
 <span><span class="co">##   If (feature 5 &lt;= 4.5)</span></span>
 <span><span class="co">##    Predict: 0.0</span></span>
 <span><span class="co">##   Else (feature 5 &gt; 4.5)</span></span>
 <span><span class="co">##    If (feature 5 &lt;= 84.5)</span></span>
 <span><span class="co">##     Predict: 1.0</span></span>
 <span><span class="co">##    Else (feature 5 &gt; 84.5)</span></span>
 <span><span class="co">##     Predict: 0.0</span></span>
 <span><span class="co">## </span></span></code></pre>
 <div class="sourceCode" id="cb134"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">dtModel</span>, <span class="va">df</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">predictions</span>, <span class="st">"Class"</span>, <span class="st">"Sex"</span>, <span class="st">"Age"</span>, <span class="st">"Freq"</span>, <span class="st">"Survived"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class    Sex   Age Freq Survived prediction</span></span>
 <span><span class="co">## 1   1st   Male Child    0       No         No</span></span>
 <span><span class="co">## 2   2nd   Male Child    0       No         No</span></span>
 <span><span class="co">## 3   3rd   Male Child   35       No        Yes</span></span>
 <span><span class="co">## 4  Crew   Male Child    0       No         No</span></span>
 <span><span class="co">## 5   1st Female Child    0       No         No</span></span>
 <span><span class="co">## 6   2nd Female Child    0       No         No</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="gradient-boosted-trees">Gradient-Boosted Trees<a class="anchor" aria-label="anchor" href="#gradient-boosted-trees"></a>
 </h4>
 <p><code>spark.gbt</code> fits a <a href="https://en.wikipedia.org/wiki/Gradient_boosting" class="external-link">gradient-boosted tree</a> classification or regression model on a <code>SparkDataFrame</code>. Users can call <code>summary</code> to get a summary of the fitted model, <code>predict</code> to make predictions, and <code>write.ml</code>/<code>read.ml</code> to save/load fitted models.</p>
 <p>We use the <code>Titanic</code> dataset to train a gradient-boosted tree and make predictions:</p>
 <div class="sourceCode" id="cb136"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="va">gbtModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.gbt.html">spark.gbt</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">Survived</span> <span class="op">~</span> <span class="va">.</span>, type <span class="op">=</span> <span class="st">"classification"</span>, maxDepth <span class="op">=</span> <span class="fl">2</span>, maxIter <span class="op">=</span> <span class="fl">2</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">gbtModel</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## Formula:  Survived ~ .</span></span>
 <span><span class="co">## Number of features:  6</span></span>
 <span><span class="co">## Features:  Class_1st Class_2nd Class_3rd Sex_Female Age_Adult Freq</span></span>
 <span><span class="co">## Feature importances:  (6,[1,2,5],[0.03336902858878361,0.16099525743106016,0.8056357139801562])</span></span>
 <span><span class="co">## Max Depth:  2</span></span>
 <span><span class="co">## Number of trees:  2</span></span>
 <span><span class="co">## Tree weights:  1 0.1</span></span>
 <span><span class="co">##  GBTClassificationModel: uid = gbtc_ace7b37d0c15, numTrees=2, numClasses=2, numFeatures=6</span></span>
 <span><span class="co">##   Tree 0 (weight 1.0):</span></span>
 <span><span class="co">##     If (feature 5 &lt;= 4.5)</span></span>
 <span><span class="co">##      If (feature 1 in {1.0})</span></span>
 <span><span class="co">##       Predict: -1.0</span></span>
 <span><span class="co">##      Else (feature 1 not in {1.0})</span></span>
 <span><span class="co">##       Predict: -0.3333333333333333</span></span>
 <span><span class="co">##     Else (feature 5 &gt; 4.5)</span></span>
 <span><span class="co">##      If (feature 5 &lt;= 84.5)</span></span>
 <span><span class="co">##       Predict: 0.5714285714285714</span></span>
 <span><span class="co">##      Else (feature 5 &gt; 84.5)</span></span>
 <span><span class="co">##       Predict: -0.42857142857142855</span></span>
 <span><span class="co">##   Tree 1 (weight 0.1):</span></span>
 <span><span class="co">##     If (feature 2 in {1.0})</span></span>
 <span><span class="co">##      If (feature 5 &lt;= 15.5)</span></span>
 <span><span class="co">##       Predict: 0.9671846896296403</span></span>
 <span><span class="co">##      Else (feature 5 &gt; 15.5)</span></span>
 <span><span class="co">##       Predict: -1.0857923804083338</span></span>
 <span><span class="co">##     Else (feature 2 not in {1.0})</span></span>
 <span><span class="co">##      If (feature 5 &lt;= 13.5)</span></span>
 <span><span class="co">##       Predict: -0.08651035613926407</span></span>
 <span><span class="co">##      Else (feature 5 &gt; 13.5)</span></span>
 <span><span class="co">##       Predict: 0.6566673506774614</span></span>
 <span><span class="co">## </span></span></code></pre>
 <div class="sourceCode" id="cb138"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">gbtModel</span>, <span class="va">df</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">predictions</span>, <span class="st">"Class"</span>, <span class="st">"Sex"</span>, <span class="st">"Age"</span>, <span class="st">"Freq"</span>, <span class="st">"Survived"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class    Sex   Age Freq Survived prediction</span></span>
 <span><span class="co">## 1   1st   Male Child    0       No         No</span></span>
 <span><span class="co">## 2   2nd   Male Child    0       No         No</span></span>
 <span><span class="co">## 3   3rd   Male Child   35       No        Yes</span></span>
 <span><span class="co">## 4  Crew   Male Child    0       No         No</span></span>
 <span><span class="co">## 5   1st Female Child    0       No         No</span></span>
 <span><span class="co">## 6   2nd Female Child    0       No         No</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="random-forest">Random Forest<a class="anchor" aria-label="anchor" href="#random-forest"></a>
 </h4>
 <p><code>spark.randomForest</code> fits a <a href="https://en.wikipedia.org/wiki/Random_forest" class="external-link">random forest</a> classification or regression model on a <code>SparkDataFrame</code>. Users can call <code>summary</code> to get a summary of the fitted model, <code>predict</code> to make predictions, and <code>write.ml</code>/<code>read.ml</code> to save/load fitted models.</p>
 <p>In the following example, we use the <code>Titanic</code> dataset to train a random forest and make predictions:</p>
 <div class="sourceCode" id="cb140"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="va">rfModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.randomForest.html">spark.randomForest</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">Survived</span> <span class="op">~</span> <span class="va">.</span>, type <span class="op">=</span> <span class="st">"classification"</span>, maxDepth <span class="op">=</span> <span class="fl">2</span>, numTrees <span class="op">=</span> <span class="fl">2</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">rfModel</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## Formula:  Survived ~ .</span></span>
 <span><span class="co">## Number of features:  6</span></span>
 <span><span class="co">## Features:  Class_1st Class_2nd Class_3rd Sex_Female Age_Adult Freq</span></span>
 <span><span class="co">## Feature importances:  (6,[3,4,5],[0.09849498327759101,0.401505016722409,0.5])</span></span>
 <span><span class="co">## Max Depth:  2</span></span>
 <span><span class="co">## Number of trees:  2</span></span>
 <span><span class="co">## Tree weights:  1 1</span></span>
 <span><span class="co">##  RandomForestClassificationModel: uid=rfc_28648dc6eca3, numTrees=2, numClasses=2, numFeatures=6</span></span>
 <span><span class="co">##   Tree 0 (weight 1.0):</span></span>
 <span><span class="co">##     If (feature 4 in {0.0})</span></span>
 <span><span class="co">##      Predict: 0.0</span></span>
 <span><span class="co">##     Else (feature 4 not in {0.0})</span></span>
 <span><span class="co">##      If (feature 3 in {1.0})</span></span>
 <span><span class="co">##       Predict: 0.0</span></span>
 <span><span class="co">##      Else (feature 3 not in {1.0})</span></span>
 <span><span class="co">##       Predict: 1.0</span></span>
 <span><span class="co">##   Tree 1 (weight 1.0):</span></span>
 <span><span class="co">##     If (feature 5 &lt;= 84.5)</span></span>
 <span><span class="co">##      If (feature 5 &lt;= 4.5)</span></span>
 <span><span class="co">##       Predict: 0.0</span></span>
 <span><span class="co">##      Else (feature 5 &gt; 4.5)</span></span>
 <span><span class="co">##       Predict: 1.0</span></span>
 <span><span class="co">##     Else (feature 5 &gt; 84.5)</span></span>
 <span><span class="co">##      Predict: 0.0</span></span>
 <span><span class="co">## </span></span></code></pre>
 <div class="sourceCode" id="cb142"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">rfModel</span>, <span class="va">df</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">predictions</span>, <span class="st">"Class"</span>, <span class="st">"Sex"</span>, <span class="st">"Age"</span>, <span class="st">"Freq"</span>, <span class="st">"Survived"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class    Sex   Age Freq Survived prediction</span></span>
 <span><span class="co">## 1   1st   Male Child    0       No         No</span></span>
 <span><span class="co">## 2   2nd   Male Child    0       No         No</span></span>
 <span><span class="co">## 3   3rd   Male Child   35       No        Yes</span></span>
 <span><span class="co">## 4  Crew   Male Child    0       No         No</span></span>
 <span><span class="co">## 5   1st Female Child    0       No         No</span></span>
 <span><span class="co">## 6   2nd Female Child    0       No         No</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="bisecting-k-means">Bisecting k-Means<a class="anchor" aria-label="anchor" href="#bisecting-k-means"></a>
 </h4>
 <p><code>spark.bisectingKmeans</code> is a kind of <a href="https://en.wikipedia.org/wiki/Hierarchical_clustering" class="external-link">hierarchical clustering</a> using a divisive (or “top-down”) approach: all observations start in one cluster, and splits are performed recursively as one moves down the hierarchy.</p>
 <div class="sourceCode" id="cb144"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">training</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.bisectingKmeans.html">spark.bisectingKmeans</a></span><span class="op">(</span><span class="va">training</span>, <span class="va">Class</span> <span class="op">~</span> <span class="va">Survived</span>, k <span class="op">=</span> <span class="fl">4</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $k</span></span>
 <span><span class="co">## [1] 4</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $coefficients</span></span>
 <span><span class="co">##   Survived_No</span></span>
 <span><span class="co">## 1 0          </span></span>
 <span><span class="co">## 2 1          </span></span>
 <span><span class="co">## 3 0          </span></span>
 <span><span class="co">## 4 1          </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $size</span></span>
 <span><span class="co">## $size[[1]]</span></span>
 <span><span class="co">## [1] 16</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $size[[2]]</span></span>
 <span><span class="co">## [1] 16</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $size[[3]]</span></span>
 <span><span class="co">## [1] 0</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $size[[4]]</span></span>
 <span><span class="co">## [1] 0</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $cluster</span></span>
 <span><span class="co">## SparkDataFrame[prediction:int]</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $is.loaded</span></span>
 <span><span class="co">## [1] FALSE</span></span></code></pre>
 <div class="sourceCode" id="cb146"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">fitted</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">training</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">fitted</span>, <span class="st">"Class"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class prediction</span></span>
 <span><span class="co">## 1   1st          1</span></span>
 <span><span class="co">## 2   2nd          1</span></span>
 <span><span class="co">## 3   3rd          1</span></span>
 <span><span class="co">## 4  Crew          1</span></span>
 <span><span class="co">## 5   1st          1</span></span>
 <span><span class="co">## 6   2nd          1</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="gaussian-mixture-model">Gaussian Mixture Model<a class="anchor" aria-label="anchor" href="#gaussian-mixture-model"></a>
 </h4>
 <p><code>spark.gaussianMixture</code> fits multivariate <a href="https://en.wikipedia.org/wiki/Mixture_model#Multivariate_Gaussian_mixture_model" class="external-link">Gaussian Mixture Model</a> (GMM) against a <code>SparkDataFrame</code>. <a href="https://en.wikipedia.org/wiki/Expectation%E2%80%93maximization_algorithm" class="external-link">Expectation-Maximization</a> (EM) is used to approximate the maximum likelihood estimator (MLE) of the model.</p>
 <p>We use a simulated example to demonstrate the usage.</p>
 <div class="sourceCode" id="cb148"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">X1</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>V1 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">4</span><span class="op">)</span>, V2 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">4</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">X2</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>V1 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">6</span>, <span class="fl">3</span><span class="op">)</span>, V2 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/stats/Normal.html" class="external-link">rnorm</a></span><span class="op">(</span><span class="fl">6</span>, <span class="fl">4</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">data</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/rbind.html">rbind</a></span><span class="op">(</span><span class="va">X1</span>, <span class="va">X2</span><span class="op">)</span></span>
 <span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">data</span><span class="op">)</span></span>
 <span><span class="va">gmmModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.gaussianMixture.html">spark.gaussianMixture</a></span><span class="op">(</span><span class="va">df</span>, <span class="op">~</span> <span class="va">V1</span> <span class="op">+</span> <span class="va">V2</span>, k <span class="op">=</span> <span class="fl">2</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">gmmModel</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $lambda</span></span>
 <span><span class="co">## [1] 0.408879 0.591121</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $mu</span></span>
 <span><span class="co">## $mu[[1]]</span></span>
 <span><span class="co">## [1] -0.8254152973  0.0009888204</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $mu[[2]]</span></span>
 <span><span class="co">## [1] 3.006119 3.620325</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $sigma</span></span>
 <span><span class="co">## $sigma[[1]]</span></span>
 <span><span class="co">##      [,1]     [,2]    </span></span>
 <span><span class="co">## [1,] 1.377944 1.092401</span></span>
 <span><span class="co">## [2,] 1.092401 1.477489</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $sigma[[2]]</span></span>
 <span><span class="co">##      [,1]      [,2]     </span></span>
 <span><span class="co">## [1,] 1.317545  0.5716919</span></span>
 <span><span class="co">## [2,] 0.5716919 0.7335671</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $loglik</span></span>
 <span><span class="co">## [1] -32.8689</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $posterior</span></span>
 <span><span class="co">## SparkDataFrame[posterior:array&lt;double&gt;]</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $is.loaded</span></span>
 <span><span class="co">## [1] FALSE</span></span></code></pre>
 <div class="sourceCode" id="cb150"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">gmmFitted</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">gmmModel</span>, <span class="va">df</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">gmmFitted</span>, <span class="st">"V1"</span>, <span class="st">"V2"</span>, <span class="st">"prediction"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##             V1         V2 prediction</span></span>
 <span><span class="co">## 1 -1.400043517  0.6215527          0</span></span>
 <span><span class="co">## 2  0.255317055  1.1484116          0</span></span>
 <span><span class="co">## 3 -2.437263611 -1.8218177          0</span></span>
 <span><span class="co">## 4 -0.005571287 -0.2473253          0</span></span>
 <span><span class="co">## 5  2.755800393  4.5124269          1</span></span>
 <span><span class="co">## 6  2.717294551  2.1369885          1</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="k-means-clustering">k-Means Clustering<a class="anchor" aria-label="anchor" href="#k-means-clustering"></a>
 </h4>
 <p><code>spark.kmeans</code> fits a <span class="math inline">\(k\)</span>-means clustering model against a <code>SparkDataFrame</code>. As an unsupervised learning method, we don’t need a response variable. Hence, the left hand side of the R formula should be left blank. The clustering is based only on the variables on the right hand side.</p>
 <div class="sourceCode" id="cb152"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">kmeansModel</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.kmeans.html">spark.kmeans</a></span><span class="op">(</span><span class="va">carsDF</span>, <span class="op">~</span> <span class="va">mpg</span> <span class="op">+</span> <span class="va">hp</span> <span class="op">+</span> <span class="va">wt</span>, k <span class="op">=</span> <span class="fl">3</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">kmeansModel</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $k</span></span>
 <span><span class="co">## [1] 3</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $coefficients</span></span>
 <span><span class="co">##        mpg        hp       wt</span></span>
 <span><span class="co">## 1 24.22353  93.52941 2.599588</span></span>
 <span><span class="co">## 2 15.80000 178.50000 3.926400</span></span>
 <span><span class="co">## 3 14.62000 263.80000 3.899000</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $size</span></span>
 <span><span class="co">## $size[[1]]</span></span>
 <span><span class="co">## [1] 17</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $size[[2]]</span></span>
 <span><span class="co">## [1] 10</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $size[[3]]</span></span>
 <span><span class="co">## [1] 5</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $cluster</span></span>
 <span><span class="co">## SparkDataFrame[prediction:int]</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $is.loaded</span></span>
 <span><span class="co">## [1] FALSE</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $clusterSize</span></span>
 <span><span class="co">## [1] 3</span></span></code></pre>
 <div class="sourceCode" id="cb154"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">kmeansPredictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">kmeansModel</span>, <span class="va">carsDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">kmeansPredictions</span>, <span class="st">"model"</span>, <span class="st">"mpg"</span>, <span class="st">"hp"</span>, <span class="st">"wt"</span>, <span class="st">"prediction"</span><span class="op">)</span>, n <span class="op">=</span> <span class="fl">20L</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##                  model  mpg  hp    wt prediction</span></span>
 <span><span class="co">## 1            Mazda RX4 21.0 110 2.620          0</span></span>
 <span><span class="co">## 2        Mazda RX4 Wag 21.0 110 2.875          0</span></span>
 <span><span class="co">## 3           Datsun 710 22.8  93 2.320          0</span></span>
 <span><span class="co">## 4       Hornet 4 Drive 21.4 110 3.215          0</span></span>
 <span><span class="co">## 5    Hornet Sportabout 18.7 175 3.440          1</span></span>
 <span><span class="co">## 6              Valiant 18.1 105 3.460          0</span></span>
 <span><span class="co">## 7           Duster 360 14.3 245 3.570          2</span></span>
 <span><span class="co">## 8            Merc 240D 24.4  62 3.190          0</span></span>
 <span><span class="co">## 9             Merc 230 22.8  95 3.150          0</span></span>
 <span><span class="co">## 10            Merc 280 19.2 123 3.440          0</span></span>
 <span><span class="co">## 11           Merc 280C 17.8 123 3.440          0</span></span>
 <span><span class="co">## 12          Merc 450SE 16.4 180 4.070          1</span></span>
 <span><span class="co">## 13          Merc 450SL 17.3 180 3.730          1</span></span>
 <span><span class="co">## 14         Merc 450SLC 15.2 180 3.780          1</span></span>
 <span><span class="co">## 15  Cadillac Fleetwood 10.4 205 5.250          1</span></span>
 <span><span class="co">## 16 Lincoln Continental 10.4 215 5.424          1</span></span>
 <span><span class="co">## 17   Chrysler Imperial 14.7 230 5.345          2</span></span>
 <span><span class="co">## 18            Fiat 128 32.4  66 2.200          0</span></span>
 <span><span class="co">## 19         Honda Civic 30.4  52 1.615          0</span></span>
 <span><span class="co">## 20      Toyota Corolla 33.9  65 1.835          0</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="latent-dirichlet-allocation">Latent Dirichlet Allocation<a class="anchor" aria-label="anchor" href="#latent-dirichlet-allocation"></a>
 </h4>
 <p><code>spark.lda</code> fits a <a href="https://en.wikipedia.org/wiki/Latent_Dirichlet_allocation" class="external-link">Latent Dirichlet Allocation</a> model on a <code>SparkDataFrame</code>. It is often used in topic modeling in which topics are inferred from a collection of text documents. LDA can be thought of as a clustering algorithm as follows:</p>
 <ul>
 <li><p>Topics correspond to cluster centers, and documents correspond to examples (rows) in a dataset.</p></li>
 <li><p>Topics and documents both exist in a feature space, where feature vectors are vectors of word counts (bag of words).</p></li>
 <li><p>Rather than clustering using a traditional distance, LDA uses a function based on a statistical model of how text documents are generated.</p></li>
 </ul>
 <p>To use LDA, we need to specify a <code>features</code> column in <code>data</code> where each entry represents a document. There are two options for the column:</p>
 <ul>
 <li><p>character string: This can be a string of the whole document. It will be parsed automatically. Additional stop words can be added in <code>customizedStopWords</code>.</p></li>
 <li><p>libSVM: Each entry is a collection of words and will be processed directly.</p></li>
 </ul>
 <p>Two more functions are provided for the fitted model.</p>
 <ul>
 <li><p><code>spark.posterior</code> returns a <code>SparkDataFrame</code> containing a column of posterior probabilities vectors named “topicDistribution”.</p></li>
 <li><p><code>spark.perplexity</code> returns the log perplexity of given <code>SparkDataFrame</code>, or the log perplexity of the training data if missing argument <code>data</code>.</p></li>
 </ul>
 <p>For more information, see the help document <code><a href="../reference/spark.lda.html">?spark.lda</a></code>.</p>
 <p>Let’s look an artificial example.</p>
 <div class="sourceCode" id="cb156"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">corpus</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>features <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span></span>
 <span>  <span class="st">"1 2 6 0 2 3 1 1 0 0 3"</span>,</span>
 <span>  <span class="st">"1 3 0 1 3 0 0 2 0 0 1"</span>,</span>
 <span>  <span class="st">"1 4 1 0 0 4 9 0 1 2 0"</span>,</span>
 <span>  <span class="st">"2 1 0 3 0 0 5 0 2 3 9"</span>,</span>
 <span>  <span class="st">"3 1 1 9 3 0 2 0 0 1 3"</span>,</span>
 <span>  <span class="st">"4 2 0 3 4 5 1 1 1 4 0"</span>,</span>
 <span>  <span class="st">"2 1 0 3 0 0 5 0 2 2 9"</span>,</span>
 <span>  <span class="st">"1 1 1 9 2 1 2 0 0 1 3"</span>,</span>
 <span>  <span class="st">"4 4 0 3 4 2 1 3 0 0 0"</span>,</span>
 <span>  <span class="st">"2 8 2 0 3 0 2 0 2 7 2"</span>,</span>
 <span>  <span class="st">"1 1 1 9 0 2 2 0 0 3 3"</span>,</span>
 <span>  <span class="st">"4 1 0 0 4 5 1 3 0 1 0"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">corpusDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">corpus</span><span class="op">)</span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.lda.html">spark.lda</a></span><span class="op">(</span>data <span class="op">=</span> <span class="va">corpusDF</span>, k <span class="op">=</span> <span class="fl">5</span>, optimizer <span class="op">=</span> <span class="st">"em"</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## $docConcentration</span></span>
 <span><span class="co">## [1] 11 11 11 11 11</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $topicConcentration</span></span>
 <span><span class="co">## [1] 1.1</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $logLikelihood</span></span>
 <span><span class="co">## [1] -353.2948</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $logPerplexity</span></span>
 <span><span class="co">## [1] 2.676476</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $isDistributed</span></span>
 <span><span class="co">## [1] TRUE</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $vocabSize</span></span>
 <span><span class="co">## [1] 10</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $topics</span></span>
 <span><span class="co">## SparkDataFrame[topic:int, term:array&lt;string&gt;, termWeights:array&lt;double&gt;]</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $vocabulary</span></span>
 <span><span class="co">##  [1] "0" "1" "2" "3" "4" "9" "5" "8" "7" "6"</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $trainingLogLikelihood</span></span>
 <span><span class="co">## [1] -239.5629</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## $logPrior</span></span>
 <span><span class="co">## [1] -980.2974</span></span></code></pre>
 <div class="sourceCode" id="cb158"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">posterior</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.lda.html">spark.posterior</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">corpusDF</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">posterior</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##                features                                     topicDistribution</span></span>
 <span><span class="co">## 1 1 2 6 0 2 3 1 1 0 0 3 0.1972180, 0.1986668, 0.2022009, 0.2006577, 0.2012566</span></span>
 <span><span class="co">## 2 1 3 0 1 3 0 0 2 0 0 1 0.1989984, 0.1988762, 0.2015961, 0.2006378, 0.1998916</span></span>
 <span><span class="co">## 3 1 4 1 0 0 4 9 0 1 2 0 0.2020604, 0.2026109, 0.1968837, 0.1987291, 0.1997158</span></span>
 <span><span class="co">## 4 2 1 0 3 0 0 5 0 2 3 9 0.2004077, 0.1981957, 0.2013008, 0.2006312, 0.1994646</span></span>
 <span><span class="co">## 5 3 1 1 9 3 0 2 0 0 1 3 0.1971485, 0.1983979, 0.2023571, 0.2011585, 0.2009380</span></span>
 <span><span class="co">## 6 4 2 0 3 4 5 1 1 1 4 0 0.2020232, 0.2041847, 0.1955393, 0.1997227, 0.1985301</span></span></code></pre>
 <div class="sourceCode" id="cb160"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">perplexity</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.lda.html">spark.perplexity</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">corpusDF</span><span class="op">)</span></span>
 <span><span class="va">perplexity</span></span></code></pre></div>
 <pre><code><span><span class="co">## [1] 2.676476</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="alternating-least-squares">Alternating Least Squares<a class="anchor" aria-label="anchor" href="#alternating-least-squares"></a>
 </h4>
 <p><code>spark.als</code> learns latent factors in <a href="https://en.wikipedia.org/wiki/Recommender_system#Collaborative_filtering" class="external-link">collaborative filtering</a> via <a href="https://dl.acm.org/doi/10.1109/MC.2009.263" class="external-link">alternating least squares</a>.</p>
 <p>There are multiple options that can be configured in <code>spark.als</code>, including <code>rank</code>, <code>reg</code>, and <code>nonnegative</code>. For a complete list, refer to the help file.</p>
 <div class="sourceCode" id="cb162"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">ratings</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">0</span>, <span class="fl">4.0</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">1</span>, <span class="fl">2.0</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">1</span>, <span class="fl">3.0</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">2</span>, <span class="fl">4.0</span><span class="op">)</span>,</span>
 <span>                <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">2</span>, <span class="fl">1</span>, <span class="fl">1.0</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">2</span>, <span class="fl">2</span>, <span class="fl">5.0</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">ratings</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"user"</span>, <span class="st">"item"</span>, <span class="st">"rating"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.als.html">spark.als</a></span><span class="op">(</span><span class="va">df</span>, <span class="st">"rating"</span>, <span class="st">"user"</span>, <span class="st">"item"</span>, rank <span class="op">=</span> <span class="fl">10</span>, reg <span class="op">=</span> <span class="fl">0.1</span>, nonnegative <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span></span></code></pre></div>
 <p>Extract latent factors.</p>
 <div class="sourceCode" id="cb163"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">stats</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">model</span><span class="op">)</span></span>
 <span><span class="va">userFactors</span> <span class="op">&lt;-</span> <span class="va">stats</span><span class="op">$</span><span class="va">userFactors</span></span>
 <span><span class="va">itemFactors</span> <span class="op">&lt;-</span> <span class="va">stats</span><span class="op">$</span><span class="va">itemFactors</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">userFactors</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">itemFactors</span><span class="op">)</span></span></code></pre></div>
 <p>Make predictions.</p>
 <div class="sourceCode" id="cb164"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">predicted</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">model</span>, <span class="va">df</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">predicted</span><span class="op">)</span></span></code></pre></div>
 </div>
 <div class="section level4">
 <h4 id="power-iteration-clustering">Power Iteration Clustering<a class="anchor" aria-label="anchor" href="#power-iteration-clustering"></a>
 </h4>
 <p>Power Iteration Clustering (PIC) is a scalable graph clustering algorithm. <code>spark.assignClusters</code> method runs the PIC algorithm and returns a cluster assignment for each input vertex.</p>
 <div class="sourceCode" id="cb165"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0L</span>, <span class="fl">1L</span>, <span class="fl">1.0</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">0L</span>, <span class="fl">2L</span>, <span class="fl">1.0</span><span class="op">)</span>,</span>
 <span>                           <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1L</span>, <span class="fl">2L</span>, <span class="fl">1.0</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">3L</span>, <span class="fl">4L</span>, <span class="fl">1.0</span><span class="op">)</span>,</span>
 <span>                           <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">4L</span>, <span class="fl">0L</span>, <span class="fl">0.1</span><span class="op">)</span><span class="op">)</span>,</span>
 <span>                      schema <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"src"</span>, <span class="st">"dst"</span>, <span class="st">"weight"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/spark.powerIterationClustering.html">spark.assignClusters</a></span><span class="op">(</span><span class="va">df</span>, initMode <span class="op">=</span> <span class="st">"degree"</span>, weightCol <span class="op">=</span> <span class="st">"weight"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   id cluster</span></span>
 <span><span class="co">## 1  4       1</span></span>
 <span><span class="co">## 2  0       0</span></span>
 <span><span class="co">## 3  1       0</span></span>
 <span><span class="co">## 4  3       1</span></span>
 <span><span class="co">## 5  2       0</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="fp-growth">FP-growth<a class="anchor" aria-label="anchor" href="#fp-growth"></a>
 </h4>
 <p><code>spark.fpGrowth</code> executes FP-growth algorithm to mine frequent itemsets on a <code>SparkDataFrame</code>. <code>itemsCol</code> should be an array of values.</p>
 <div class="sourceCode" id="cb167"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/selectExpr.html">selectExpr</a></span><span class="op">(</span><span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>rawItems <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span></span>
 <span>  <span class="st">"T,R,U"</span>, <span class="st">"T,S"</span>, <span class="st">"V,R"</span>, <span class="st">"R,U,T,V"</span>, <span class="st">"R,S"</span>, <span class="st">"V,S,U"</span>, <span class="st">"U,R"</span>, <span class="st">"S,T"</span>, <span class="st">"V,R"</span>, <span class="st">"V,U,S"</span>,</span>
 <span>  <span class="st">"T,V,U"</span>, <span class="st">"R,V"</span>, <span class="st">"T,S"</span>, <span class="st">"T,S"</span>, <span class="st">"S,T"</span>, <span class="st">"S,U"</span>, <span class="st">"T,R"</span>, <span class="st">"V,R"</span>, <span class="st">"S,V"</span>, <span class="st">"T,S,U"</span></span>
 <span><span class="op">)</span><span class="op">)</span><span class="op">)</span>, <span class="st">"split(rawItems, ',') AS items"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="va">fpm</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.fpGrowth.html">spark.fpGrowth</a></span><span class="op">(</span><span class="va">df</span>, minSupport <span class="op">=</span> <span class="fl">0.2</span>, minConfidence <span class="op">=</span> <span class="fl">0.5</span><span class="op">)</span></span></code></pre></div>
 <p><code>spark.freqItemsets</code> method can be used to retrieve a <code>SparkDataFrame</code> with the frequent itemsets.</p>
 <div class="sourceCode" id="cb168"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/spark.fpGrowth.html">spark.freqItemsets</a></span><span class="op">(</span><span class="va">fpm</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   items freq</span></span>
 <span><span class="co">## 1     S   11</span></span>
 <span><span class="co">## 2     T   10</span></span>
 <span><span class="co">## 3  T, S    6</span></span>
 <span><span class="co">## 4     R    9</span></span>
 <span><span class="co">## 5     V    9</span></span>
 <span><span class="co">## 6  V, R    5</span></span></code></pre>
 <p><code>spark.associationRules</code> returns a <code>SparkDataFrame</code> with the association rules.</p>
 <div class="sourceCode" id="cb170"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/spark.fpGrowth.html">spark.associationRules</a></span><span class="op">(</span><span class="va">fpm</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   antecedent consequent confidence      lift support</span></span>
 <span><span class="co">## 1          V          R  0.5555556 1.2345679    0.25</span></span>
 <span><span class="co">## 2          S          T  0.5454545 1.0909091    0.30</span></span>
 <span><span class="co">## 3          T          S  0.6000000 1.0909091    0.30</span></span>
 <span><span class="co">## 4          R          V  0.5555556 1.2345679    0.25</span></span>
 <span><span class="co">## 5          U          S  0.5000000 0.9090909    0.20</span></span>
 <span><span class="co">## 6          U          T  0.5000000 1.0000000    0.20</span></span></code></pre>
 <p>We can make predictions based on the <code>antecedent</code>.</p>
 <div class="sourceCode" id="cb172"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">fpm</span>, <span class="va">df</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##        items prediction</span></span>
 <span><span class="co">## 1    T, R, U       S, V</span></span>
 <span><span class="co">## 2       T, S       NULL</span></span>
 <span><span class="co">## 3       V, R       NULL</span></span>
 <span><span class="co">## 4 R, U, T, V          S</span></span>
 <span><span class="co">## 5       R, S       T, V</span></span>
 <span><span class="co">## 6    V, S, U       R, T</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="prefixspan">PrefixSpan<a class="anchor" aria-label="anchor" href="#prefixspan"></a>
 </h4>
 <p><code>spark.findFrequentSequentialPatterns</code> method can be used to find the complete set of frequent sequential patterns in the input sequences of itemsets.</p>
 <div class="sourceCode" id="cb174"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1L</span>, <span class="fl">2L</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">3L</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>,</span>
 <span>                           <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1L</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">3L</span>, <span class="fl">2L</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1L</span>, <span class="fl">2L</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>,</span>
 <span>                           <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">1L</span>, <span class="fl">2L</span><span class="op">)</span>, <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">5L</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>,</span>
 <span>                           <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="fl">6L</span><span class="op">)</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>,</span>
 <span>                      schema <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"sequence"</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/spark.prefixSpan.html">spark.findFrequentSequentialPatterns</a></span><span class="op">(</span><span class="va">df</span>, minSupport <span class="op">=</span> <span class="fl">0.5</span>, maxPatternLength <span class="op">=</span> <span class="fl">5L</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   sequence freq</span></span>
 <span><span class="co">## 1        1    3</span></span>
 <span><span class="co">## 2        3    2</span></span>
 <span><span class="co">## 3        2    3</span></span>
 <span><span class="co">## 4     1, 2    3</span></span>
 <span><span class="co">## 5     1, 3    2</span></span></code></pre>
 </div>
 <div class="section level4">
 <h4 id="kolmogorov-smirnov-test">Kolmogorov-Smirnov Test<a class="anchor" aria-label="anchor" href="#kolmogorov-smirnov-test"></a>
 </h4>
 <p><code>spark.kstest</code> runs a two-sided, one-sample <a href="https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test" class="external-link">Kolmogorov-Smirnov (KS) test</a>. Given a <code>SparkDataFrame</code>, the test compares continuous data in a given column <code>testCol</code> with the theoretical distribution specified by parameter <code>nullHypothesis</code>. Users can call <code>summary</code> to get a summary of the test results.</p>
 <p>In the following example, we test whether the <code>Titanic</code> dataset’s <code>Freq</code> column follows a normal distribution. We set the parameters of the normal distribution using the mean and standard deviation of the sample.</p>
 <div class="sourceCode" id="cb176"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="va">freqStats</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="va">df</span>, <span class="fu"><a href="../reference/column_aggregate_functions.html">mean</a></span><span class="op">(</span><span class="va">df</span><span class="op">$</span><span class="va">Freq</span><span class="op">)</span>, <span class="fu"><a href="../reference/column_aggregate_functions.html">sd</a></span><span class="op">(</span><span class="va">df</span><span class="op">$</span><span class="va">Freq</span><span class="op">)</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">freqMean</span> <span class="op">&lt;-</span> <span class="va">freqStats</span><span class="op">[</span><span class="fl">1</span><span class="op">]</span></span>
 <span><span class="va">freqStd</span> <span class="op">&lt;-</span> <span class="va">freqStats</span><span class="op">[</span><span class="fl">2</span><span class="op">]</span></span>
 <span></span>
 <span><span class="va">test</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.kstest.html">spark.kstest</a></span><span class="op">(</span><span class="va">df</span>, <span class="st">"Freq"</span>, <span class="st">"norm"</span>, <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="va">freqMean</span>, <span class="va">freqStd</span><span class="op">)</span><span class="op">)</span></span>
 <span><span class="va">testSummary</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">test</span><span class="op">)</span></span>
 <span><span class="va">testSummary</span></span></code></pre></div>
 <pre><code><span><span class="co">## Kolmogorov-Smirnov test summary:</span></span>
 <span><span class="co">## degrees of freedom = 0 </span></span>
 <span><span class="co">## statistic = 0.3065126710255011 </span></span>
 <span><span class="co">## pValue = 0.0036336792155329256 </span></span>
 <span><span class="co">## Very strong presumption against null hypothesis: Sample follows theoretical distribution.</span></span></code></pre>
 </div>
 </div>
 <div class="section level3">
 <h3 id="model-persistence">Model Persistence<a class="anchor" aria-label="anchor" href="#model-persistence"></a>
 </h3>
 <p>The following example shows how to save/load an ML model in SparkR.</p>
 <div class="sourceCode" id="cb178"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">t</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/as.data.frame.html">as.data.frame</a></span><span class="op">(</span><span class="va">Titanic</span><span class="op">)</span></span>
 <span><span class="va">training</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/createDataFrame.html">createDataFrame</a></span><span class="op">(</span><span class="va">t</span><span class="op">)</span></span>
 <span><span class="va">gaussianGLM</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/spark.glm.html">spark.glm</a></span><span class="op">(</span><span class="va">training</span>, <span class="va">Freq</span> <span class="op">~</span> <span class="va">Sex</span> <span class="op">+</span> <span class="va">Age</span>, family <span class="op">=</span> <span class="st">"gaussian"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Save and then load a fitted MLlib model</span></span>
 <span><span class="va">modelPath</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>pattern <span class="op">=</span> <span class="st">"ml"</span>, fileext <span class="op">=</span> <span class="st">".tmp"</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/write.ml.html">write.ml</a></span><span class="op">(</span><span class="va">gaussianGLM</span>, <span class="va">modelPath</span><span class="op">)</span></span>
 <span><span class="va">gaussianGLM2</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/read.ml.html">read.ml</a></span><span class="op">(</span><span class="va">modelPath</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Check model summary</span></span>
 <span><span class="fu"><a href="../reference/summary.html">summary</a></span><span class="op">(</span><span class="va">gaussianGLM2</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">## </span></span>
 <span><span class="co">## Saved-loaded model does not support output 'Deviance Residuals'.</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Coefficients:</span></span>
 <span><span class="co">##              Estimate  Std. Error  t value   Pr(&gt;|t|)</span></span>
 <span><span class="co">## (Intercept)    46.219      35.994   1.2841  0.2092846</span></span>
 <span><span class="co">## Sex_Female    -78.812      41.562  -1.8962  0.0679311</span></span>
 <span><span class="co">## Age_Adult     123.938      41.562   2.9820  0.0057522</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## (Dispersion parameter for gaussian family taken to be 13819.52)</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">##     Null deviance: 573341  on 31  degrees of freedom</span></span>
 <span><span class="co">## Residual deviance: 400766  on 29  degrees of freedom</span></span>
 <span><span class="co">## AIC: 400.7</span></span>
 <span><span class="co">## </span></span>
 <span><span class="co">## Number of Fisher Scoring iterations: 1</span></span></code></pre>
 <div class="sourceCode" id="cb180"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="co"># Check model prediction</span></span>
 <span><span class="va">gaussianPredictions</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/predict.html">predict</a></span><span class="op">(</span><span class="va">gaussianGLM2</span>, <span class="va">training</span><span class="op">)</span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="va">gaussianPredictions</span><span class="op">)</span></span></code></pre></div>
 <pre><code><span><span class="co">##   Class    Sex   Age Survived Freq label prediction</span></span>
 <span><span class="co">## 1   1st   Male Child       No    0     0   46.21875</span></span>
 <span><span class="co">## 2   2nd   Male Child       No    0     0   46.21875</span></span>
 <span><span class="co">## 3   3rd   Male Child       No   35    35   46.21875</span></span>
 <span><span class="co">## 4  Crew   Male Child       No    0     0   46.21875</span></span>
 <span><span class="co">## 5   1st Female Child       No    0     0  -32.59375</span></span>
 <span><span class="co">## 6   2nd Female Child       No    0     0  -32.59375</span></span></code></pre>
 <div class="sourceCode" id="cb182"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/unlink.html" class="external-link">unlink</a></span><span class="op">(</span><span class="va">modelPath</span><span class="op">)</span></span></code></pre></div>
 </div>
 </div>
 <div class="section level2">
 <h2 id="structured-streaming">Structured Streaming<a class="anchor" aria-label="anchor" href="#structured-streaming"></a>
 </h2>
 <p>SparkR supports the Structured Streaming API.</p>
 <p>You can check the Structured Streaming Programming Guide for <a href="https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#programming-model" class="external-link">an introduction</a> to its programming model and basic concepts.</p>
 <div class="section level3">
 <h3 id="simple-source-and-sink">Simple Source and Sink<a class="anchor" aria-label="anchor" href="#simple-source-and-sink"></a>
 </h3>
 <p>Spark has a few built-in input sources. As an example, to test with a socket source reading text into words and displaying the computed word counts:</p>
 <div class="sourceCode" id="cb183"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="co"># Create DataFrame representing the stream of input lines from connection</span></span>
 <span><span class="va">lines</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/read.stream.html">read.stream</a></span><span class="op">(</span><span class="st">"socket"</span>, host <span class="op">=</span> <span class="va">hostname</span>, port <span class="op">=</span> <span class="va">port</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Split the lines into words</span></span>
 <span><span class="va">words</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/selectExpr.html">selectExpr</a></span><span class="op">(</span><span class="va">lines</span>, <span class="st">"explode(split(value, ' ')) as word"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Generate running word count</span></span>
 <span><span class="va">wordCounts</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/count.html">count</a></span><span class="op">(</span><span class="fu"><a href="../reference/groupBy.html">groupBy</a></span><span class="op">(</span><span class="va">words</span>, <span class="st">"word"</span><span class="op">)</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Start running the query that prints the running counts to the console</span></span>
 <span><span class="va">query</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/write.stream.html">write.stream</a></span><span class="op">(</span><span class="va">wordCounts</span>, <span class="st">"console"</span>, outputMode <span class="op">=</span> <span class="st">"complete"</span><span class="op">)</span></span></code></pre></div>
 </div>
 <div class="section level3">
 <h3 id="kafka-source">Kafka Source<a class="anchor" aria-label="anchor" href="#kafka-source"></a>
 </h3>
 <p>It is simple to read data from Kafka. For more information, see <a href="https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#input-sources" class="external-link">Input Sources</a> supported by Structured Streaming.</p>
 <div class="sourceCode" id="cb184"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">topic</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/read.stream.html">read.stream</a></span><span class="op">(</span><span class="st">"kafka"</span>,</span>
 <span>                     kafka.bootstrap.servers <span class="op">=</span> <span class="st">"host1:port1,host2:port2"</span>,</span>
 <span>                     subscribe <span class="op">=</span> <span class="st">"topic1"</span><span class="op">)</span></span>
 <span><span class="va">keyvalue</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/selectExpr.html">selectExpr</a></span><span class="op">(</span><span class="va">topic</span>, <span class="st">"CAST(key AS STRING)"</span>, <span class="st">"CAST(value AS STRING)"</span><span class="op">)</span></span></code></pre></div>
 </div>
 <div class="section level3">
 <h3 id="operations-and-sinks">Operations and Sinks<a class="anchor" aria-label="anchor" href="#operations-and-sinks"></a>
 </h3>
 <p>Most of the common operations on <code>SparkDataFrame</code> are supported for streaming, including selection, projection, and aggregation. Once you have defined the final result, to start the streaming computation, you will call the <code>write.stream</code> method setting a sink and <code>outputMode</code>.</p>
 <p>A streaming <code>SparkDataFrame</code> can be written for debugging to the console, to a temporary in-memory table, or for further processing in a fault-tolerant manner to a File Sink in different formats.</p>
 <div class="sourceCode" id="cb185"><pre class="downlit sourceCode r">
 <code class="sourceCode R"><span><span class="va">noAggDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/select.html">select</a></span><span class="op">(</span><span class="fu"><a href="../reference/filter.html">where</a></span><span class="op">(</span><span class="va">deviceDataStreamingDf</span>, <span class="st">"signal &gt; 10"</span><span class="op">)</span>, <span class="st">"device"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Print new data to console</span></span>
 <span><span class="fu"><a href="../reference/write.stream.html">write.stream</a></span><span class="op">(</span><span class="va">noAggDF</span>, <span class="st">"console"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Write new data to Parquet files</span></span>
 <span><span class="fu"><a href="../reference/write.stream.html">write.stream</a></span><span class="op">(</span><span class="va">noAggDF</span>,</span>
 <span>             <span class="st">"parquet"</span>,</span>
 <span>             path <span class="op">=</span> <span class="st">"path/to/destination/dir"</span>,</span>
 <span>             checkpointLocation <span class="op">=</span> <span class="st">"path/to/checkpoint/dir"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Aggregate</span></span>
 <span><span class="va">aggDF</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/count.html">count</a></span><span class="op">(</span><span class="fu"><a href="../reference/groupBy.html">groupBy</a></span><span class="op">(</span><span class="va">noAggDF</span>, <span class="st">"device"</span><span class="op">)</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Print updated aggregations to console</span></span>
 <span><span class="fu"><a href="../reference/write.stream.html">write.stream</a></span><span class="op">(</span><span class="va">aggDF</span>, <span class="st">"console"</span>, outputMode <span class="op">=</span> <span class="st">"complete"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="co"># Have all the aggregates in an in memory table. The query name will be the table name</span></span>
 <span><span class="fu"><a href="../reference/write.stream.html">write.stream</a></span><span class="op">(</span><span class="va">aggDF</span>, <span class="st">"memory"</span>, queryName <span class="op">=</span> <span class="st">"aggregates"</span>, outputMode <span class="op">=</span> <span class="st">"complete"</span><span class="op">)</span></span>
 <span></span>
 <span><span class="fu"><a href="../reference/head.html">head</a></span><span class="op">(</span><span class="fu"><a href="../reference/sql.html">sql</a></span><span class="op">(</span><span class="st">"select * from aggregates"</span><span class="op">)</span><span class="op">)</span></span></code></pre></div>
 </div>
 </div>
 <div class="section level2">
 <h2 id="advanced-topics">Advanced Topics<a class="anchor" aria-label="anchor" href="#advanced-topics"></a>
 </h2>
 <div class="section level3">
 <h3 id="sparkr-object-classes">SparkR Object Classes<a class="anchor" aria-label="anchor" href="#sparkr-object-classes"></a>
 </h3>
 <p>There are three main object classes in SparkR you may be working with.</p>
 <ul>
 <li>
 <p><code>SparkDataFrame</code>: the central component of SparkR. It is an S4 class representing distributed collection of data organized into named columns, which is conceptually equivalent to a table in a relational database or a data frame in R. It has two slots <code>sdf</code> and <code>env</code>.</p>
 <ul>
 <li>
 <code>sdf</code> stores a reference to the corresponding Spark Dataset in the Spark JVM backend.</li>
 <li>
 <code>env</code> saves the meta-information of the object such as <code>isCached</code>.</li>
 </ul>
 <p>It can be created by data import methods or by transforming an existing <code>SparkDataFrame</code>. We can manipulate <code>SparkDataFrame</code> by numerous data processing functions and feed that into machine learning algorithms.</p>
 </li>
 <li>
 <p><code>Column</code>: an S4 class representing a column of <code>SparkDataFrame</code>. The slot <code>jc</code> saves a reference to the corresponding <code>Column</code> object in the Spark JVM backend.</p>
 <p>It can be obtained from a <code>SparkDataFrame</code> by <code>$</code> operator, e.g., <code>df$col</code>. More often, it is used together with other functions, for example, with <code>select</code> to select particular columns, with <code>filter</code> and constructed conditions to select rows, with aggregation functions to compute aggregate statistics for each group.</p>
 </li>
 <li>
 <p><code>GroupedData</code>: an S4 class representing grouped data created by <code>groupBy</code> or by transforming other <code>GroupedData</code>. Its <code>sgd</code> slot saves a reference to a <code>RelationalGroupedDataset</code> object in the backend.</p>
 <p>This is often an intermediate object with group information and followed up by aggregation operations.</p>
 </li>
 </ul>
 </div>
 <div class="section level3">
 <h3 id="architecture">Architecture<a class="anchor" aria-label="anchor" href="#architecture"></a>
 </h3>
 <p>A complete description of architecture can be seen in the references, in particular the paper <em>SparkR: Scaling R Programs with Spark</em>.</p>
 <p>Under the hood of SparkR is Spark SQL engine. This avoids the overheads of running interpreted R code, and the optimized SQL execution engine in Spark uses structural information about data and computation flow to perform a bunch of optimizations to speed up the computation.</p>
 <p>The main method calls of actual computation happen in the Spark JVM of the driver. We have a socket-based SparkR API that allows us to invoke functions on the JVM from R. We use a SparkR JVM backend that listens on a Netty-based socket server.</p>
 <p>Two kinds of RPCs are supported in the SparkR JVM backend: method invocation and creating new objects. Method invocation can be done in two ways.</p>
 <ul>
 <li><p><code>sparkR.callJMethod</code> takes a reference to an existing Java object and a list of arguments to be passed on to the method.</p></li>
 <li><p><code>sparkR.callJStatic</code> takes a class name for static method and a list of arguments to be passed on to the method.</p></li>
 </ul>
 <p>The arguments are serialized using our custom wire format which is then deserialized on the JVM side. We then use Java reflection to invoke the appropriate method.</p>
 <p>To create objects, <code>sparkR.newJObject</code> is used and then similarly the appropriate constructor is invoked with provided arguments.</p>
 <p>Finally, we use a new R class <code>jobj</code> that refers to a Java object existing in the backend. These references are tracked on the Java side and are automatically garbage collected when they go out of scope on the R side.</p>
 </div>
 </div>
 <div class="section level2">
 <h2 id="appendix">Appendix<a class="anchor" aria-label="anchor" href="#appendix"></a>
 </h2>
 <div class="section level3">
 <h3 id="DataTypes">R and Spark Data Types<a class="anchor" aria-label="anchor" href="#DataTypes"></a>
 </h3>
 <table class="table">
 <thead><tr class="header">
 <th>R</th>
 <th>Spark</th>
 </tr></thead>
 <tbody>
 <tr class="odd">
 <td>byte</td>
 <td>byte</td>
 </tr>
 <tr class="even">
 <td>integer</td>
 <td>integer</td>
 </tr>
 <tr class="odd">
 <td>float</td>
 <td>float</td>
 </tr>
 <tr class="even">
 <td>double</td>
 <td>double</td>
 </tr>
 <tr class="odd">
 <td>numeric</td>
 <td>double</td>
 </tr>
 <tr class="even">
 <td>character</td>
 <td>string</td>
 </tr>
 <tr class="odd">
 <td>string</td>
 <td>string</td>
 </tr>
 <tr class="even">
 <td>binary</td>
 <td>binary</td>
 </tr>
 <tr class="odd">
 <td>raw</td>
 <td>binary</td>
 </tr>
 <tr class="even">
 <td>logical</td>
 <td>boolean</td>
 </tr>
 <tr class="odd">
 <td>POSIXct</td>
 <td>timestamp</td>
 </tr>
 <tr class="even">
 <td>POSIXlt</td>
 <td>timestamp</td>
 </tr>
 <tr class="odd">
 <td>Date</td>
 <td>date</td>
 </tr>
 <tr class="even">
 <td>array</td>
 <td>array</td>
 </tr>
 <tr class="odd">
 <td>list</td>
 <td>array</td>
 </tr>
 <tr class="even">
 <td>env</td>
 <td>map</td>
 </tr>
 </tbody>
 </table>
 </div>
 </div>
 <div class="section level2">
 <h2 id="references">References<a class="anchor" aria-label="anchor" href="#references"></a>
 </h2>
 <ul>
 <li><p><a href="https://spark.apache.org/docs/latest/cluster-overview.html" class="external-link">Spark Cluster Mode Overview</a></p></li>
 <li><p><a href="https://spark.apache.org/docs/latest/submitting-applications.html" class="external-link">Submitting Spark Applications</a></p></li>
 <li><p><a href="https://spark.apache.org/docs/latest/ml-guide.html" class="external-link">Machine Learning Library Guide (MLlib)</a></p></li>
 <li><p><a href="https://people.csail.mit.edu/matei/papers/2016/sigmod_sparkr.pdf" class="external-link">SparkR: Scaling R Programs with Spark</a>, Shivaram Venkataraman, Zongheng Yang, Davies Liu, Eric Liang, Hossein Falaki, Xiangrui Meng, Reynold Xin, Ali Ghodsi, Michael Franklin, Ion Stoica, and Matei Zaharia. SIGMOD 2016. June 2016.</p></li>
 </ul>
 </div>
   </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
     </nav></aside>
 </div>


     <footer><div class="copyright">
   <p></p>
 <p>Developed by <a href="https://www.apache.org/" class="external-link"> The Apache Software Foundation</a>.</p>
 </div>

 <div class="pkgdown">
   <p></p>
 <p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.9.</p>
   <p class="preferably">Using <a href="https://preferably.amirmasoudabdol.name/?source=footer" class="external-link">preferably</a> template.</p>
 </div>

     </footer>
 </div>


   </body>
 </html>