Google Git
Sign in
apache / spark-website / 28656f2c00baf1bc2d26570a16b4f96f9a890723 / . / site / docs / 2.2.1 / ml-tuning.html
blob: 84eb34d783cb5f0276812ebc691bb9f8aa4c6942 [file] [log] [blame]
<!DOCTYPE html>
<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
<!--[if IE 8]> <html class="no-js lt-ie9"> <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<title>ML Tuning - Spark 2.2.1 Documentation</title>
<link rel="stylesheet" href="css/bootstrap.min.css">
<style>
body {
padding-top: 60px;
padding-bottom: 40px;
}
</style>
<meta name="viewport" content="width=device-width">
<link rel="stylesheet" href="css/bootstrap-responsive.min.css">
<link rel="stylesheet" href="css/main.css">
<script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script>
<link rel="stylesheet" href="css/pygments-default.css">
<!-- Google analytics script -->
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-32518208-2']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
</head>
<body>
<!--[if lt IE 7]>
<p class="chromeframe">You are using an outdated browser. <a href="http://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p>
<![endif]-->
<!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html -->
<div class="navbar navbar-fixed-top" id="topbar">
<div class="navbar-inner">
<div class="container">
<div class="brand"><a href="index.html">
<img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">2.2.1</span>
</div>
<ul class="nav">
<!--TODO(andyk): Add class="active" attribute to li some how.-->
<li><a href="index.html">Overview</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Programming Guides<b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="quick-start.html">Quick Start</a></li>
<li><a href="rdd-programming-guide.html">RDDs, Accumulators, Broadcasts Vars</a></li>
<li><a href="sql-programming-guide.html">SQL, DataFrames, and Datasets</a></li>
<li><a href="structured-streaming-programming-guide.html">Structured Streaming</a></li>
<li><a href="streaming-programming-guide.html">Spark Streaming (DStreams)</a></li>
<li><a href="ml-guide.html">MLlib (Machine Learning)</a></li>
<li><a href="graphx-programming-guide.html">GraphX (Graph Processing)</a></li>
<li><a href="sparkr.html">SparkR (R on Spark)</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="api/scala/index.html#org.apache.spark.package">Scala</a></li>
<li><a href="api/java/index.html">Java</a></li>
<li><a href="api/python/index.html">Python</a></li>
<li><a href="api/R/index.html">R</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Deploying<b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="cluster-overview.html">Overview</a></li>
<li><a href="submitting-applications.html">Submitting Applications</a></li>
<li class="divider"></li>
<li><a href="spark-standalone.html">Spark Standalone</a></li>
<li><a href="running-on-mesos.html">Mesos</a></li>
<li><a href="running-on-yarn.html">YARN</a></li>
</ul>
</li>
<li class="dropdown">
<a href="api.html" class="dropdown-toggle" data-toggle="dropdown">More<b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="configuration.html">Configuration</a></li>
<li><a href="monitoring.html">Monitoring</a></li>
<li><a href="tuning.html">Tuning Guide</a></li>
<li><a href="job-scheduling.html">Job Scheduling</a></li>
<li><a href="security.html">Security</a></li>
<li><a href="hardware-provisioning.html">Hardware Provisioning</a></li>
<li class="divider"></li>
<li><a href="building-spark.html">Building Spark</a></li>
<li><a href="http://spark.apache.org/contributing.html">Contributing to Spark</a></li>
<li><a href="http://spark.apache.org/third-party-projects.html">Third Party Projects</a></li>
</ul>
</li>
</ul>
<!--<p class="navbar-text pull-right"><span class="version-text">v2.2.1</span></p>-->
</div>
</div>
</div>
<div class="container-wrapper">
<div class="left-menu-wrapper">
<div class="left-menu">
<h3><a href="ml-guide.html">MLlib: Main Guide</a></h3>
<ul>
<li>
<a href="ml-statistics.html">
Basic statistics
</a>
</li>
<li>
<a href="ml-pipeline.html">
Pipelines
</a>
</li>
<li>
<a href="ml-features.html">
Extracting, transforming and selecting features
</a>
</li>
<li>
<a href="ml-classification-regression.html">
Classification and Regression
</a>
</li>
<li>
<a href="ml-clustering.html">
Clustering
</a>
</li>
<li>
<a href="ml-collaborative-filtering.html">
Collaborative filtering
</a>
</li>
<li>
<a href="ml-frequent-pattern-mining.html">
Frequent Pattern Mining
</a>
</li>
<li>
<a href="ml-tuning.html">
<b>Model selection and tuning</b>
</a>
</li>
<li>
<a href="ml-advanced.html">
Advanced topics
</a>
</li>
</ul>
<h3><a href="mllib-guide.html">MLlib: RDD-based API Guide</a></h3>
<ul>
<li>
<a href="mllib-data-types.html">
Data types
</a>
</li>
<li>
<a href="mllib-statistics.html">
Basic statistics
</a>
</li>
<li>
<a href="mllib-classification-regression.html">
Classification and regression
</a>
</li>
<li>
<a href="mllib-collaborative-filtering.html">
Collaborative filtering
</a>
</li>
<li>
<a href="mllib-clustering.html">
Clustering
</a>
</li>
<li>
<a href="mllib-dimensionality-reduction.html">
Dimensionality reduction
</a>
</li>
<li>
<a href="mllib-feature-extraction.html">
Feature extraction and transformation
</a>
</li>
<li>
<a href="mllib-frequent-pattern-mining.html">
Frequent pattern mining
</a>
</li>
<li>
<a href="mllib-evaluation-metrics.html">
Evaluation metrics
</a>
</li>
<li>
<a href="mllib-pmml-model-export.html">
PMML model export
</a>
</li>
<li>
<a href="mllib-optimization.html">
Optimization (developer)
</a>
</li>
</ul>
</div>
</div>
<input id="nav-trigger" class="nav-trigger" checked type="checkbox">
<label for="nav-trigger"></label>
<div class="content-with-sidebar" id="content">
<h1 class="title">ML Tuning: model selection and hyperparameter tuning</h1>
<p><code>\[
\newcommand{\R}{\mathbb{R}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\x}{\mathbf{x}}
\newcommand{\y}{\mathbf{y}}
\newcommand{\wv}{\mathbf{w}}
\newcommand{\av}{\mathbf{\alpha}}
\newcommand{\bv}{\mathbf{b}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\id}{\mathbf{I}}
\newcommand{\ind}{\mathbf{1}}
\newcommand{\0}{\mathbf{0}}
\newcommand{\unit}{\mathbf{e}}
\newcommand{\one}{\mathbf{1}}
\newcommand{\zero}{\mathbf{0}}
\]</code></p>
<p>This section describes how to use MLlib&#8217;s tooling for tuning ML algorithms and Pipelines.
Built-in Cross-Validation and other tooling allow users to optimize hyperparameters in algorithms and Pipelines.</p>
<p><strong>Table of contents</strong></p>
<ul id="markdown-toc">
<li><a href="#model-selection-aka-hyperparameter-tuning" id="markdown-toc-model-selection-aka-hyperparameter-tuning">Model selection (a.k.a. hyperparameter tuning)</a></li>
<li><a href="#cross-validation" id="markdown-toc-cross-validation">Cross-Validation</a></li>
<li><a href="#train-validation-split" id="markdown-toc-train-validation-split">Train-Validation Split</a></li>
</ul>
<h1 id="model-selection-aka-hyperparameter-tuning">Model selection (a.k.a. hyperparameter tuning)</h1>
<p>An important task in ML is <em>model selection</em>, or using data to find the best model or parameters for a given task. This is also called <em>tuning</em>.
Tuning may be done for individual <code>Estimator</code>s such as <code>LogisticRegression</code>, or for entire <code>Pipeline</code>s which include multiple algorithms, featurization, and other steps. Users can tune an entire <code>Pipeline</code> at once, rather than tuning each element in the <code>Pipeline</code> separately.</p>
<p>MLlib supports model selection using tools such as <a href="api/scala/index.html#org.apache.spark.ml.tuning.CrossValidator"><code>CrossValidator</code></a> and <a href="api/scala/index.html#org.apache.spark.ml.tuning.TrainValidationSplit"><code>TrainValidationSplit</code></a>.
These tools require the following items:</p>
<ul>
<li><a href="api/scala/index.html#org.apache.spark.ml.Estimator"><code>Estimator</code></a>: algorithm or <code>Pipeline</code> to tune</li>
<li>Set of <code>ParamMap</code>s: parameters to choose from, sometimes called a &#8220;parameter grid&#8221; to search over</li>
<li><a href="api/scala/index.html#org.apache.spark.ml.evaluation.Evaluator"><code>Evaluator</code></a>: metric to measure how well a fitted <code>Model</code> does on held-out test data</li>
</ul>
<p>At a high level, these model selection tools work as follows:</p>
<ul>
<li>They split the input data into separate training and test datasets.</li>
<li>For each (training, test) pair, they iterate through the set of <code>ParamMap</code>s:
<ul>
<li>For each <code>ParamMap</code>, they fit the <code>Estimator</code> using those parameters, get the fitted <code>Model</code>, and evaluate the <code>Model</code>&#8217;s performance using the <code>Evaluator</code>.</li>
</ul>
</li>
<li>They select the <code>Model</code> produced by the best-performing set of parameters.</li>
</ul>
<p>The <code>Evaluator</code> can be a <a href="api/scala/index.html#org.apache.spark.ml.evaluation.RegressionEvaluator"><code>RegressionEvaluator</code></a>
for regression problems, a <a href="api/scala/index.html#org.apache.spark.ml.evaluation.BinaryClassificationEvaluator"><code>BinaryClassificationEvaluator</code></a>
for binary data, or a <a href="api/scala/index.html#org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator"><code>MulticlassClassificationEvaluator</code></a>
for multiclass problems. The default metric used to choose the best <code>ParamMap</code> can be overridden by the <code>setMetricName</code>
method in each of these evaluators.</p>
<p>To help construct the parameter grid, users can use the <a href="api/scala/index.html#org.apache.spark.ml.tuning.ParamGridBuilder"><code>ParamGridBuilder</code></a> utility.</p>
<h1 id="cross-validation">Cross-Validation</h1>
<p><code>CrossValidator</code> begins by splitting the dataset into a set of <em>folds</em> which are used as separate training and test datasets. E.g., with <code>$k=3$</code> folds, <code>CrossValidator</code> will generate 3 (training, test) dataset pairs, each of which uses 2/3 of the data for training and 1/3 for testing. To evaluate a particular <code>ParamMap</code>, <code>CrossValidator</code> computes the average evaluation metric for the 3 <code>Model</code>s produced by fitting the <code>Estimator</code> on the 3 different (training, test) dataset pairs.</p>
<p>After identifying the best <code>ParamMap</code>, <code>CrossValidator</code> finally re-fits the <code>Estimator</code> using the best <code>ParamMap</code> and the entire dataset.</p>
<p><strong>Examples: model selection via cross-validation</strong></p>
<p>The following example demonstrates using <code>CrossValidator</code> to select from a grid of parameters.</p>
<p>Note that cross-validation over a grid of parameters is expensive.
E.g., in the example below, the parameter grid has 3 values for <code>hashingTF.numFeatures</code> and 2 values for <code>lr.regParam</code>, and <code>CrossValidator</code> uses 2 folds. This multiplies out to <code>$(3 \times 2) \times 2 = 12$</code> different models being trained.
In realistic settings, it can be common to try many more parameters and use more folds (<code>$k=3$</code> and <code>$k=10$</code> are common).
In other words, using <code>CrossValidator</code> can be very expensive.
However, it is also a well-established method for choosing parameters which is more statistically sound than heuristic hand-tuning.</p>
<div class="codetabs">
<div data-lang="scala">
<p>Refer to the <a href="api/scala/index.html#org.apache.spark.ml.tuning.CrossValidator"><code>CrossValidator</code> Scala docs</a> for details on the API.</p>
<div class="highlight"><pre><span></span><span class="k">import</span> <span class="nn">org.apache.spark.ml.Pipeline</span>
<span class="k">import</span> <span class="nn">org.apache.spark.ml.classification.LogisticRegression</span>
<span class="k">import</span> <span class="nn">org.apache.spark.ml.evaluation.BinaryClassificationEvaluator</span>
<span class="k">import</span> <span class="nn">org.apache.spark.ml.feature.</span><span class="o">{</span><span class="nc">HashingTF</span><span class="o">,</span> <span class="nc">Tokenizer</span><span class="o">}</span>
<span class="k">import</span> <span class="nn">org.apache.spark.ml.linalg.Vector</span>
<span class="k">import</span> <span class="nn">org.apache.spark.ml.tuning.</span><span class="o">{</span><span class="nc">CrossValidator</span><span class="o">,</span> <span class="nc">ParamGridBuilder</span><span class="o">}</span>
<span class="k">import</span> <span class="nn">org.apache.spark.sql.Row</span>
<span class="c1">// Prepare training data from a list of (id, text, label) tuples.</span>
<span class="k">val</span> <span class="n">training</span> <span class="k">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="o">(</span><span class="nc">Seq</span><span class="o">(</span>
<span class="o">(</span><span class="mi">0L</span><span class="o">,</span> <span class="s">&quot;a b c d e spark&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">1L</span><span class="o">,</span> <span class="s">&quot;b d&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">2L</span><span class="o">,</span> <span class="s">&quot;spark f g h&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">3L</span><span class="o">,</span> <span class="s">&quot;hadoop mapreduce&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">4L</span><span class="o">,</span> <span class="s">&quot;b spark who&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">5L</span><span class="o">,</span> <span class="s">&quot;g d a y&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">6L</span><span class="o">,</span> <span class="s">&quot;spark fly&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">7L</span><span class="o">,</span> <span class="s">&quot;was mapreduce&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">8L</span><span class="o">,</span> <span class="s">&quot;e spark program&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">9L</span><span class="o">,</span> <span class="s">&quot;a e c l&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">10L</span><span class="o">,</span> <span class="s">&quot;spark compile&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="o">(</span><span class="mi">11L</span><span class="o">,</span> <span class="s">&quot;hadoop software&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">)</span>
<span class="o">)).</span><span class="n">toDF</span><span class="o">(</span><span class="s">&quot;id&quot;</span><span class="o">,</span> <span class="s">&quot;text&quot;</span><span class="o">,</span> <span class="s">&quot;label&quot;</span><span class="o">)</span>
<span class="c1">// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.</span>
<span class="k">val</span> <span class="n">tokenizer</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">Tokenizer</span><span class="o">()</span>
<span class="o">.</span><span class="n">setInputCol</span><span class="o">(</span><span class="s">&quot;text&quot;</span><span class="o">)</span>
<span class="o">.</span><span class="n">setOutputCol</span><span class="o">(</span><span class="s">&quot;words&quot;</span><span class="o">)</span>
<span class="k">val</span> <span class="n">hashingTF</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">HashingTF</span><span class="o">()</span>
<span class="o">.</span><span class="n">setInputCol</span><span class="o">(</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">getOutputCol</span><span class="o">)</span>
<span class="o">.</span><span class="n">setOutputCol</span><span class="o">(</span><span class="s">&quot;features&quot;</span><span class="o">)</span>
<span class="k">val</span> <span class="n">lr</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">LogisticRegression</span><span class="o">()</span>
<span class="o">.</span><span class="n">setMaxIter</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
<span class="k">val</span> <span class="n">pipeline</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">Pipeline</span><span class="o">()</span>
<span class="o">.</span><span class="n">setStages</span><span class="o">(</span><span class="nc">Array</span><span class="o">(</span><span class="n">tokenizer</span><span class="o">,</span> <span class="n">hashingTF</span><span class="o">,</span> <span class="n">lr</span><span class="o">))</span>
<span class="c1">// We use a ParamGridBuilder to construct a grid of parameters to search over.</span>
<span class="c1">// With 3 values for hashingTF.numFeatures and 2 values for lr.regParam,</span>
<span class="c1">// this grid will have 3 x 2 = 6 parameter settings for CrossValidator to choose from.</span>
<span class="k">val</span> <span class="n">paramGrid</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">ParamGridBuilder</span><span class="o">()</span>
<span class="o">.</span><span class="n">addGrid</span><span class="o">(</span><span class="n">hashingTF</span><span class="o">.</span><span class="n">numFeatures</span><span class="o">,</span> <span class="nc">Array</span><span class="o">(</span><span class="mi">10</span><span class="o">,</span> <span class="mi">100</span><span class="o">,</span> <span class="mi">1000</span><span class="o">))</span>
<span class="o">.</span><span class="n">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="n">regParam</span><span class="o">,</span> <span class="nc">Array</span><span class="o">(</span><span class="mf">0.1</span><span class="o">,</span> <span class="mf">0.01</span><span class="o">))</span>
<span class="o">.</span><span class="n">build</span><span class="o">()</span>
<span class="c1">// We now treat the Pipeline as an Estimator, wrapping it in a CrossValidator instance.</span>
<span class="c1">// This will allow us to jointly choose parameters for all Pipeline stages.</span>
<span class="c1">// A CrossValidator requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.</span>
<span class="c1">// Note that the evaluator here is a BinaryClassificationEvaluator and its default metric</span>
<span class="c1">// is areaUnderROC.</span>
<span class="k">val</span> <span class="n">cv</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">CrossValidator</span><span class="o">()</span>
<span class="o">.</span><span class="n">setEstimator</span><span class="o">(</span><span class="n">pipeline</span><span class="o">)</span>
<span class="o">.</span><span class="n">setEvaluator</span><span class="o">(</span><span class="k">new</span> <span class="nc">BinaryClassificationEvaluator</span><span class="o">)</span>
<span class="o">.</span><span class="n">setEstimatorParamMaps</span><span class="o">(</span><span class="n">paramGrid</span><span class="o">)</span>
<span class="o">.</span><span class="n">setNumFolds</span><span class="o">(</span><span class="mi">2</span><span class="o">)</span> <span class="c1">// Use 3+ in practice</span>
<span class="c1">// Run cross-validation, and choose the best set of parameters.</span>
<span class="k">val</span> <span class="n">cvModel</span> <span class="k">=</span> <span class="n">cv</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">training</span><span class="o">)</span>
<span class="c1">// Prepare test documents, which are unlabeled (id, text) tuples.</span>
<span class="k">val</span> <span class="n">test</span> <span class="k">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="o">(</span><span class="nc">Seq</span><span class="o">(</span>
<span class="o">(</span><span class="mi">4L</span><span class="o">,</span> <span class="s">&quot;spark i j k&quot;</span><span class="o">),</span>
<span class="o">(</span><span class="mi">5L</span><span class="o">,</span> <span class="s">&quot;l m n&quot;</span><span class="o">),</span>
<span class="o">(</span><span class="mi">6L</span><span class="o">,</span> <span class="s">&quot;mapreduce spark&quot;</span><span class="o">),</span>
<span class="o">(</span><span class="mi">7L</span><span class="o">,</span> <span class="s">&quot;apache hadoop&quot;</span><span class="o">)</span>
<span class="o">)).</span><span class="n">toDF</span><span class="o">(</span><span class="s">&quot;id&quot;</span><span class="o">,</span> <span class="s">&quot;text&quot;</span><span class="o">)</span>
<span class="c1">// Make predictions on test documents. cvModel uses the best model found (lrModel).</span>
<span class="n">cvModel</span><span class="o">.</span><span class="n">transform</span><span class="o">(</span><span class="n">test</span><span class="o">)</span>
<span class="o">.</span><span class="n">select</span><span class="o">(</span><span class="s">&quot;id&quot;</span><span class="o">,</span> <span class="s">&quot;text&quot;</span><span class="o">,</span> <span class="s">&quot;probability&quot;</span><span class="o">,</span> <span class="s">&quot;prediction&quot;</span><span class="o">)</span>
<span class="o">.</span><span class="n">collect</span><span class="o">()</span>
<span class="o">.</span><span class="n">foreach</span> <span class="o">{</span> <span class="k">case</span> <span class="nc">Row</span><span class="o">(</span><span class="n">id</span><span class="k">:</span> <span class="kt">Long</span><span class="o">,</span> <span class="n">text</span><span class="k">:</span> <span class="kt">String</span><span class="o">,</span> <span class="n">prob</span><span class="k">:</span> <span class="kt">Vector</span><span class="o">,</span> <span class="n">prediction</span><span class="k">:</span> <span class="kt">Double</span><span class="o">)</span> <span class="k">=&gt;</span>
<span class="n">println</span><span class="o">(</span><span class="s">s&quot;(</span><span class="si">$id</span><span class="s">, </span><span class="si">$text</span><span class="s">) --&gt; prob=</span><span class="si">$prob</span><span class="s">, prediction=</span><span class="si">$prediction</span><span class="s">&quot;</span><span class="o">)</span>
<span class="o">}</span>
</pre></div>
<div><small>Find full example code at "examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala" in the Spark repo.</small></div>
</div>
<div data-lang="java">
<p>Refer to the <a href="api/java/org/apache/spark/ml/tuning/CrossValidator.html"><code>CrossValidator</code> Java docs</a> for details on the API.</p>
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">java.util.Arrays</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.Pipeline</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.PipelineStage</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.classification.LogisticRegression</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.evaluation.BinaryClassificationEvaluator</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.feature.HashingTF</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.feature.Tokenizer</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.param.ParamMap</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.tuning.CrossValidator</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.tuning.CrossValidatorModel</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.tuning.ParamGridBuilder</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.sql.Dataset</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.sql.Row</span><span class="o">;</span>
<span class="c1">// Prepare training documents, which are labeled.</span>
<span class="n">Dataset</span><span class="o">&lt;</span><span class="n">Row</span><span class="o">&gt;</span> <span class="n">training</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="na">createDataFrame</span><span class="o">(</span><span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">0</span><span class="n">L</span><span class="o">,</span> <span class="s">&quot;a b c d e spark&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">1L</span><span class="o">,</span> <span class="s">&quot;b d&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">2L</span><span class="o">,</span><span class="s">&quot;spark f g h&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">3L</span><span class="o">,</span> <span class="s">&quot;hadoop mapreduce&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">4L</span><span class="o">,</span> <span class="s">&quot;b spark who&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">5L</span><span class="o">,</span> <span class="s">&quot;g d a y&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">6L</span><span class="o">,</span> <span class="s">&quot;spark fly&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">7L</span><span class="o">,</span> <span class="s">&quot;was mapreduce&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">8L</span><span class="o">,</span> <span class="s">&quot;e spark program&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">9L</span><span class="o">,</span> <span class="s">&quot;a e c l&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">10L</span><span class="o">,</span> <span class="s">&quot;spark compile&quot;</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaLabeledDocument</span><span class="o">(</span><span class="mi">11L</span><span class="o">,</span> <span class="s">&quot;hadoop software&quot;</span><span class="o">,</span> <span class="mf">0.0</span><span class="o">)</span>
<span class="o">),</span> <span class="n">JavaLabeledDocument</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
<span class="c1">// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.</span>
<span class="n">Tokenizer</span> <span class="n">tokenizer</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Tokenizer</span><span class="o">()</span>
<span class="o">.</span><span class="na">setInputCol</span><span class="o">(</span><span class="s">&quot;text&quot;</span><span class="o">)</span>
<span class="o">.</span><span class="na">setOutputCol</span><span class="o">(</span><span class="s">&quot;words&quot;</span><span class="o">);</span>
<span class="n">HashingTF</span> <span class="n">hashingTF</span> <span class="o">=</span> <span class="k">new</span> <span class="n">HashingTF</span><span class="o">()</span>
<span class="o">.</span><span class="na">setNumFeatures</span><span class="o">(</span><span class="mi">1000</span><span class="o">)</span>
<span class="o">.</span><span class="na">setInputCol</span><span class="o">(</span><span class="n">tokenizer</span><span class="o">.</span><span class="na">getOutputCol</span><span class="o">())</span>
<span class="o">.</span><span class="na">setOutputCol</span><span class="o">(</span><span class="s">&quot;features&quot;</span><span class="o">);</span>
<span class="n">LogisticRegression</span> <span class="n">lr</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LogisticRegression</span><span class="o">()</span>
<span class="o">.</span><span class="na">setMaxIter</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
<span class="o">.</span><span class="na">setRegParam</span><span class="o">(</span><span class="mf">0.01</span><span class="o">);</span>
<span class="n">Pipeline</span> <span class="n">pipeline</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Pipeline</span><span class="o">()</span>
<span class="o">.</span><span class="na">setStages</span><span class="o">(</span><span class="k">new</span> <span class="n">PipelineStage</span><span class="o">[]</span> <span class="o">{</span><span class="n">tokenizer</span><span class="o">,</span> <span class="n">hashingTF</span><span class="o">,</span> <span class="n">lr</span><span class="o">});</span>
<span class="c1">// We use a ParamGridBuilder to construct a grid of parameters to search over.</span>
<span class="c1">// With 3 values for hashingTF.numFeatures and 2 values for lr.regParam,</span>
<span class="c1">// this grid will have 3 x 2 = 6 parameter settings for CrossValidator to choose from.</span>
<span class="n">ParamMap</span><span class="o">[]</span> <span class="n">paramGrid</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ParamGridBuilder</span><span class="o">()</span>
<span class="o">.</span><span class="na">addGrid</span><span class="o">(</span><span class="n">hashingTF</span><span class="o">.</span><span class="na">numFeatures</span><span class="o">(),</span> <span class="k">new</span> <span class="kt">int</span><span class="o">[]</span> <span class="o">{</span><span class="mi">10</span><span class="o">,</span> <span class="mi">100</span><span class="o">,</span> <span class="mi">1000</span><span class="o">})</span>
<span class="o">.</span><span class="na">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="na">regParam</span><span class="o">(),</span> <span class="k">new</span> <span class="kt">double</span><span class="o">[]</span> <span class="o">{</span><span class="mf">0.1</span><span class="o">,</span> <span class="mf">0.01</span><span class="o">})</span>
<span class="o">.</span><span class="na">build</span><span class="o">();</span>
<span class="c1">// We now treat the Pipeline as an Estimator, wrapping it in a CrossValidator instance.</span>
<span class="c1">// This will allow us to jointly choose parameters for all Pipeline stages.</span>
<span class="c1">// A CrossValidator requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.</span>
<span class="c1">// Note that the evaluator here is a BinaryClassificationEvaluator and its default metric</span>
<span class="c1">// is areaUnderROC.</span>
<span class="n">CrossValidator</span> <span class="n">cv</span> <span class="o">=</span> <span class="k">new</span> <span class="n">CrossValidator</span><span class="o">()</span>
<span class="o">.</span><span class="na">setEstimator</span><span class="o">(</span><span class="n">pipeline</span><span class="o">)</span>
<span class="o">.</span><span class="na">setEvaluator</span><span class="o">(</span><span class="k">new</span> <span class="n">BinaryClassificationEvaluator</span><span class="o">())</span>
<span class="o">.</span><span class="na">setEstimatorParamMaps</span><span class="o">(</span><span class="n">paramGrid</span><span class="o">).</span><span class="na">setNumFolds</span><span class="o">(</span><span class="mi">2</span><span class="o">);</span> <span class="c1">// Use 3+ in practice</span>
<span class="c1">// Run cross-validation, and choose the best set of parameters.</span>
<span class="n">CrossValidatorModel</span> <span class="n">cvModel</span> <span class="o">=</span> <span class="n">cv</span><span class="o">.</span><span class="na">fit</span><span class="o">(</span><span class="n">training</span><span class="o">);</span>
<span class="c1">// Prepare test documents, which are unlabeled.</span>
<span class="n">Dataset</span><span class="o">&lt;</span><span class="n">Row</span><span class="o">&gt;</span> <span class="n">test</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="na">createDataFrame</span><span class="o">(</span><span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span>
<span class="k">new</span> <span class="n">JavaDocument</span><span class="o">(</span><span class="mi">4L</span><span class="o">,</span> <span class="s">&quot;spark i j k&quot;</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaDocument</span><span class="o">(</span><span class="mi">5L</span><span class="o">,</span> <span class="s">&quot;l m n&quot;</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaDocument</span><span class="o">(</span><span class="mi">6L</span><span class="o">,</span> <span class="s">&quot;mapreduce spark&quot;</span><span class="o">),</span>
<span class="k">new</span> <span class="n">JavaDocument</span><span class="o">(</span><span class="mi">7L</span><span class="o">,</span> <span class="s">&quot;apache hadoop&quot;</span><span class="o">)</span>
<span class="o">),</span> <span class="n">JavaDocument</span><span class="o">.</span><span class="na">class</span><span class="o">);</span>
<span class="c1">// Make predictions on test documents. cvModel uses the best model found (lrModel).</span>
<span class="n">Dataset</span><span class="o">&lt;</span><span class="n">Row</span><span class="o">&gt;</span> <span class="n">predictions</span> <span class="o">=</span> <span class="n">cvModel</span><span class="o">.</span><span class="na">transform</span><span class="o">(</span><span class="n">test</span><span class="o">);</span>
<span class="k">for</span> <span class="o">(</span><span class="n">Row</span> <span class="n">r</span> <span class="o">:</span> <span class="n">predictions</span><span class="o">.</span><span class="na">select</span><span class="o">(</span><span class="s">&quot;id&quot;</span><span class="o">,</span> <span class="s">&quot;text&quot;</span><span class="o">,</span> <span class="s">&quot;probability&quot;</span><span class="o">,</span> <span class="s">&quot;prediction&quot;</span><span class="o">).</span><span class="na">collectAsList</span><span class="o">())</span> <span class="o">{</span>
<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">&quot;(&quot;</span> <span class="o">+</span> <span class="n">r</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> <span class="o">+</span> <span class="s">&quot;, &quot;</span> <span class="o">+</span> <span class="n">r</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> <span class="o">+</span> <span class="s">&quot;) --&gt; prob=&quot;</span> <span class="o">+</span> <span class="n">r</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="mi">2</span><span class="o">)</span>
<span class="o">+</span> <span class="s">&quot;, prediction=&quot;</span> <span class="o">+</span> <span class="n">r</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="mi">3</span><span class="o">));</span>
<span class="o">}</span>
</pre></div>
<div><small>Find full example code at "examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java" in the Spark repo.</small></div>
</div>
<div data-lang="python">
<p>Refer to the <a href="api/python/pyspark.ml.html#pyspark.ml.tuning.CrossValidator"><code>CrossValidator</code> Python docs</a> for more details on the API.</p>
<div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">pyspark.ml</span> <span class="kn">import</span> <span class="n">Pipeline</span>
<span class="kn">from</span> <span class="nn">pyspark.ml.classification</span> <span class="kn">import</span> <span class="n">LogisticRegression</span>
<span class="kn">from</span> <span class="nn">pyspark.ml.evaluation</span> <span class="kn">import</span> <span class="n">BinaryClassificationEvaluator</span>
<span class="kn">from</span> <span class="nn">pyspark.ml.feature</span> <span class="kn">import</span> <span class="n">HashingTF</span><span class="p">,</span> <span class="n">Tokenizer</span>
<span class="kn">from</span> <span class="nn">pyspark.ml.tuning</span> <span class="kn">import</span> <span class="n">CrossValidator</span><span class="p">,</span> <span class="n">ParamGridBuilder</span>
<span class="c1"># Prepare training documents, which are labeled.</span>
<span class="n">training</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([</span>
<span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;a b c d e spark&quot;</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;b d&quot;</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="s2">&quot;spark f g h&quot;</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="s2">&quot;hadoop mapreduce&quot;</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="s2">&quot;b spark who&quot;</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="s2">&quot;g d a y&quot;</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="s2">&quot;spark fly&quot;</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">7</span><span class="p">,</span> <span class="s2">&quot;was mapreduce&quot;</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="s2">&quot;e spark program&quot;</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">9</span><span class="p">,</span> <span class="s2">&quot;a e c l&quot;</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="s2">&quot;spark compile&quot;</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">),</span>
<span class="p">(</span><span class="mi">11</span><span class="p">,</span> <span class="s2">&quot;hadoop software&quot;</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">)</span>
<span class="p">],</span> <span class="p">[</span><span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="s2">&quot;text&quot;</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">])</span>
<span class="c1"># Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.</span>
<span class="n">tokenizer</span> <span class="o">=</span> <span class="n">Tokenizer</span><span class="p">(</span><span class="n">inputCol</span><span class="o">=</span><span class="s2">&quot;text&quot;</span><span class="p">,</span> <span class="n">outputCol</span><span class="o">=</span><span class="s2">&quot;words&quot;</span><span class="p">)</span>
<span class="n">hashingTF</span> <span class="o">=</span> <span class="n">HashingTF</span><span class="p">(</span><span class="n">inputCol</span><span class="o">=</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">getOutputCol</span><span class="p">(),</span> <span class="n">outputCol</span><span class="o">=</span><span class="s2">&quot;features&quot;</span><span class="p">)</span>
<span class="n">lr</span> <span class="o">=</span> <span class="n">LogisticRegression</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="n">pipeline</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="p">(</span><span class="n">stages</span><span class="o">=</span><span class="p">[</span><span class="n">tokenizer</span><span class="p">,</span> <span class="n">hashingTF</span><span class="p">,</span> <span class="n">lr</span><span class="p">])</span>
<span class="c1"># We now treat the Pipeline as an Estimator, wrapping it in a CrossValidator instance.</span>
<span class="c1"># This will allow us to jointly choose parameters for all Pipeline stages.</span>
<span class="c1"># A CrossValidator requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.</span>
<span class="c1"># We use a ParamGridBuilder to construct a grid of parameters to search over.</span>
<span class="c1"># With 3 values for hashingTF.numFeatures and 2 values for lr.regParam,</span>
<span class="c1"># this grid will have 3 x 2 = 6 parameter settings for CrossValidator to choose from.</span>
<span class="n">paramGrid</span> <span class="o">=</span> <span class="n">ParamGridBuilder</span><span class="p">()</span> \
<span class="o">.</span><span class="n">addGrid</span><span class="p">(</span><span class="n">hashingTF</span><span class="o">.</span><span class="n">numFeatures</span><span class="p">,</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">1000</span><span class="p">])</span> \
<span class="o">.</span><span class="n">addGrid</span><span class="p">(</span><span class="n">lr</span><span class="o">.</span><span class="n">regParam</span><span class="p">,</span> <span class="p">[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.01</span><span class="p">])</span> \
<span class="o">.</span><span class="n">build</span><span class="p">()</span>
<span class="n">crossval</span> <span class="o">=</span> <span class="n">CrossValidator</span><span class="p">(</span><span class="n">estimator</span><span class="o">=</span><span class="n">pipeline</span><span class="p">,</span>
<span class="n">estimatorParamMaps</span><span class="o">=</span><span class="n">paramGrid</span><span class="p">,</span>
<span class="n">evaluator</span><span class="o">=</span><span class="n">BinaryClassificationEvaluator</span><span class="p">(),</span>
<span class="n">numFolds</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="c1"># use 3+ folds in practice</span>
<span class="c1"># Run cross-validation, and choose the best set of parameters.</span>
<span class="n">cvModel</span> <span class="o">=</span> <span class="n">crossval</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">training</span><span class="p">)</span>
<span class="c1"># Prepare test documents, which are unlabeled.</span>
<span class="n">test</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">([</span>
<span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="s2">&quot;spark i j k&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="s2">&quot;l m n&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="s2">&quot;mapreduce spark&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="mi">7</span><span class="p">,</span> <span class="s2">&quot;apache hadoop&quot;</span><span class="p">)</span>
<span class="p">],</span> <span class="p">[</span><span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="s2">&quot;text&quot;</span><span class="p">])</span>
<span class="c1"># Make predictions on test documents. cvModel uses the best model found (lrModel).</span>
<span class="n">prediction</span> <span class="o">=</span> <span class="n">cvModel</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">test</span><span class="p">)</span>
<span class="n">selected</span> <span class="o">=</span> <span class="n">prediction</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="s2">&quot;text&quot;</span><span class="p">,</span> <span class="s2">&quot;probability&quot;</span><span class="p">,</span> <span class="s2">&quot;prediction&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">selected</span><span class="o">.</span><span class="n">collect</span><span class="p">():</span>
<span class="k">print</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
</pre></div>
<div><small>Find full example code at "examples/src/main/python/ml/cross_validator.py" in the Spark repo.</small></div>
</div>
</div>
<h1 id="train-validation-split">Train-Validation Split</h1>
<p>In addition to <code>CrossValidator</code> Spark also offers <code>TrainValidationSplit</code> for hyper-parameter tuning.
<code>TrainValidationSplit</code> only evaluates each combination of parameters once, as opposed to k times in
the case of <code>CrossValidator</code>. It is therefore less expensive,
but will not produce as reliable results when the training dataset is not sufficiently large.</p>
<p>Unlike <code>CrossValidator</code>, <code>TrainValidationSplit</code> creates a single (training, test) dataset pair.
It splits the dataset into these two parts using the <code>trainRatio</code> parameter. For example with <code>$trainRatio=0.75$</code>,
<code>TrainValidationSplit</code> will generate a training and test dataset pair where 75% of the data is used for training and 25% for validation.</p>
<p>Like <code>CrossValidator</code>, <code>TrainValidationSplit</code> finally fits the <code>Estimator</code> using the best <code>ParamMap</code> and the entire dataset.</p>
<p><strong>Examples: model selection via train validation split</strong></p>
<div class="codetabs">
<div data-lang="scala">
<p>Refer to the <a href="api/scala/index.html#org.apache.spark.ml.tuning.TrainValidationSplit"><code>TrainValidationSplit</code> Scala docs</a> for details on the API.</p>
<div class="highlight"><pre><span></span><span class="k">import</span> <span class="nn">org.apache.spark.ml.evaluation.RegressionEvaluator</span>
<span class="k">import</span> <span class="nn">org.apache.spark.ml.regression.LinearRegression</span>
<span class="k">import</span> <span class="nn">org.apache.spark.ml.tuning.</span><span class="o">{</span><span class="nc">ParamGridBuilder</span><span class="o">,</span> <span class="nc">TrainValidationSplit</span><span class="o">}</span>
<span class="c1">// Prepare training and test data.</span>
<span class="k">val</span> <span class="n">data</span> <span class="k">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">format</span><span class="o">(</span><span class="s">&quot;libsvm&quot;</span><span class="o">).</span><span class="n">load</span><span class="o">(</span><span class="s">&quot;data/mllib/sample_linear_regression_data.txt&quot;</span><span class="o">)</span>
<span class="k">val</span> <span class="nc">Array</span><span class="o">(</span><span class="n">training</span><span class="o">,</span> <span class="n">test</span><span class="o">)</span> <span class="k">=</span> <span class="n">data</span><span class="o">.</span><span class="n">randomSplit</span><span class="o">(</span><span class="nc">Array</span><span class="o">(</span><span class="mf">0.9</span><span class="o">,</span> <span class="mf">0.1</span><span class="o">),</span> <span class="n">seed</span> <span class="k">=</span> <span class="mi">12345</span><span class="o">)</span>
<span class="k">val</span> <span class="n">lr</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">LinearRegression</span><span class="o">()</span>
<span class="o">.</span><span class="n">setMaxIter</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
<span class="c1">// We use a ParamGridBuilder to construct a grid of parameters to search over.</span>
<span class="c1">// TrainValidationSplit will try all combinations of values and determine best model using</span>
<span class="c1">// the evaluator.</span>
<span class="k">val</span> <span class="n">paramGrid</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">ParamGridBuilder</span><span class="o">()</span>
<span class="o">.</span><span class="n">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="n">regParam</span><span class="o">,</span> <span class="nc">Array</span><span class="o">(</span><span class="mf">0.1</span><span class="o">,</span> <span class="mf">0.01</span><span class="o">))</span>
<span class="o">.</span><span class="n">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="n">fitIntercept</span><span class="o">)</span>
<span class="o">.</span><span class="n">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="n">elasticNetParam</span><span class="o">,</span> <span class="nc">Array</span><span class="o">(</span><span class="mf">0.0</span><span class="o">,</span> <span class="mf">0.5</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">))</span>
<span class="o">.</span><span class="n">build</span><span class="o">()</span>
<span class="c1">// In this case the estimator is simply the linear regression.</span>
<span class="c1">// A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.</span>
<span class="k">val</span> <span class="n">trainValidationSplit</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">TrainValidationSplit</span><span class="o">()</span>
<span class="o">.</span><span class="n">setEstimator</span><span class="o">(</span><span class="n">lr</span><span class="o">)</span>
<span class="o">.</span><span class="n">setEvaluator</span><span class="o">(</span><span class="k">new</span> <span class="nc">RegressionEvaluator</span><span class="o">)</span>
<span class="o">.</span><span class="n">setEstimatorParamMaps</span><span class="o">(</span><span class="n">paramGrid</span><span class="o">)</span>
<span class="c1">// 80% of the data will be used for training and the remaining 20% for validation.</span>
<span class="o">.</span><span class="n">setTrainRatio</span><span class="o">(</span><span class="mf">0.8</span><span class="o">)</span>
<span class="c1">// Run train validation split, and choose the best set of parameters.</span>
<span class="k">val</span> <span class="n">model</span> <span class="k">=</span> <span class="n">trainValidationSplit</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">training</span><span class="o">)</span>
<span class="c1">// Make predictions on test data. model is the model with combination of parameters</span>
<span class="c1">// that performed best.</span>
<span class="n">model</span><span class="o">.</span><span class="n">transform</span><span class="o">(</span><span class="n">test</span><span class="o">)</span>
<span class="o">.</span><span class="n">select</span><span class="o">(</span><span class="s">&quot;features&quot;</span><span class="o">,</span> <span class="s">&quot;label&quot;</span><span class="o">,</span> <span class="s">&quot;prediction&quot;</span><span class="o">)</span>
<span class="o">.</span><span class="n">show</span><span class="o">()</span>
</pre></div>
<div><small>Find full example code at "examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala" in the Spark repo.</small></div>
</div>
<div data-lang="java">
<p>Refer to the <a href="api/java/org/apache/spark/ml/tuning/TrainValidationSplit.html"><code>TrainValidationSplit</code> Java docs</a> for details on the API.</p>
<div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">org.apache.spark.ml.evaluation.RegressionEvaluator</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.param.ParamMap</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.regression.LinearRegression</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.tuning.ParamGridBuilder</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.tuning.TrainValidationSplit</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.ml.tuning.TrainValidationSplitModel</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.sql.Dataset</span><span class="o">;</span>
<span class="kn">import</span> <span class="nn">org.apache.spark.sql.Row</span><span class="o">;</span>
<span class="n">Dataset</span><span class="o">&lt;</span><span class="n">Row</span><span class="o">&gt;</span> <span class="n">data</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">format</span><span class="o">(</span><span class="s">&quot;libsvm&quot;</span><span class="o">)</span>
<span class="o">.</span><span class="na">load</span><span class="o">(</span><span class="s">&quot;data/mllib/sample_linear_regression_data.txt&quot;</span><span class="o">);</span>
<span class="c1">// Prepare training and test data.</span>
<span class="n">Dataset</span><span class="o">&lt;</span><span class="n">Row</span><span class="o">&gt;[]</span> <span class="n">splits</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="na">randomSplit</span><span class="o">(</span><span class="k">new</span> <span class="kt">double</span><span class="o">[]</span> <span class="o">{</span><span class="mf">0.9</span><span class="o">,</span> <span class="mf">0.1</span><span class="o">},</span> <span class="mi">12345</span><span class="o">);</span>
<span class="n">Dataset</span><span class="o">&lt;</span><span class="n">Row</span><span class="o">&gt;</span> <span class="n">training</span> <span class="o">=</span> <span class="n">splits</span><span class="o">[</span><span class="mi">0</span><span class="o">];</span>
<span class="n">Dataset</span><span class="o">&lt;</span><span class="n">Row</span><span class="o">&gt;</span> <span class="n">test</span> <span class="o">=</span> <span class="n">splits</span><span class="o">[</span><span class="mi">1</span><span class="o">];</span>
<span class="n">LinearRegression</span> <span class="n">lr</span> <span class="o">=</span> <span class="k">new</span> <span class="n">LinearRegression</span><span class="o">();</span>
<span class="c1">// We use a ParamGridBuilder to construct a grid of parameters to search over.</span>
<span class="c1">// TrainValidationSplit will try all combinations of values and determine best model using</span>
<span class="c1">// the evaluator.</span>
<span class="n">ParamMap</span><span class="o">[]</span> <span class="n">paramGrid</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ParamGridBuilder</span><span class="o">()</span>
<span class="o">.</span><span class="na">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="na">regParam</span><span class="o">(),</span> <span class="k">new</span> <span class="kt">double</span><span class="o">[]</span> <span class="o">{</span><span class="mf">0.1</span><span class="o">,</span> <span class="mf">0.01</span><span class="o">})</span>
<span class="o">.</span><span class="na">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="na">fitIntercept</span><span class="o">())</span>
<span class="o">.</span><span class="na">addGrid</span><span class="o">(</span><span class="n">lr</span><span class="o">.</span><span class="na">elasticNetParam</span><span class="o">(),</span> <span class="k">new</span> <span class="kt">double</span><span class="o">[]</span> <span class="o">{</span><span class="mf">0.0</span><span class="o">,</span> <span class="mf">0.5</span><span class="o">,</span> <span class="mf">1.0</span><span class="o">})</span>
<span class="o">.</span><span class="na">build</span><span class="o">();</span>
<span class="c1">// In this case the estimator is simply the linear regression.</span>
<span class="c1">// A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.</span>
<span class="n">TrainValidationSplit</span> <span class="n">trainValidationSplit</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TrainValidationSplit</span><span class="o">()</span>
<span class="o">.</span><span class="na">setEstimator</span><span class="o">(</span><span class="n">lr</span><span class="o">)</span>
<span class="o">.</span><span class="na">setEvaluator</span><span class="o">(</span><span class="k">new</span> <span class="n">RegressionEvaluator</span><span class="o">())</span>
<span class="o">.</span><span class="na">setEstimatorParamMaps</span><span class="o">(</span><span class="n">paramGrid</span><span class="o">)</span>
<span class="o">.</span><span class="na">setTrainRatio</span><span class="o">(</span><span class="mf">0.8</span><span class="o">);</span> <span class="c1">// 80% for training and the remaining 20% for validation</span>
<span class="c1">// Run train validation split, and choose the best set of parameters.</span>
<span class="n">TrainValidationSplitModel</span> <span class="n">model</span> <span class="o">=</span> <span class="n">trainValidationSplit</span><span class="o">.</span><span class="na">fit</span><span class="o">(</span><span class="n">training</span><span class="o">);</span>
<span class="c1">// Make predictions on test data. model is the model with combination of parameters</span>
<span class="c1">// that performed best.</span>
<span class="n">model</span><span class="o">.</span><span class="na">transform</span><span class="o">(</span><span class="n">test</span><span class="o">)</span>
<span class="o">.</span><span class="na">select</span><span class="o">(</span><span class="s">&quot;features&quot;</span><span class="o">,</span> <span class="s">&quot;label&quot;</span><span class="o">,</span> <span class="s">&quot;prediction&quot;</span><span class="o">)</span>
<span class="o">.</span><span class="na">show</span><span class="o">();</span>
</pre></div>
<div><small>Find full example code at "examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java" in the Spark repo.</small></div>
</div>
<div data-lang="python">
<p>Refer to the <a href="api/python/pyspark.ml.html#pyspark.ml.tuning.TrainValidationSplit"><code>TrainValidationSplit</code> Python docs</a> for more details on the API.</p>
<div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">pyspark.ml.evaluation</span> <span class="kn">import</span> <span class="n">RegressionEvaluator</span>
<span class="kn">from</span> <span class="nn">pyspark.ml.regression</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
<span class="kn">from</span> <span class="nn">pyspark.ml.tuning</span> <span class="kn">import</span> <span class="n">ParamGridBuilder</span><span class="p">,</span> <span class="n">TrainValidationSplit</span>
<span class="c1"># Prepare training and test data.</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">spark</span><span class="o">.</span><span class="n">read</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s2">&quot;libsvm&quot;</span><span class="p">)</span>\
<span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s2">&quot;data/mllib/sample_linear_regression_data.txt&quot;</span><span class="p">)</span>
<span class="n">train</span><span class="p">,</span> <span class="n">test</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">randomSplit</span><span class="p">([</span><span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">],</span> <span class="n">seed</span><span class="o">=</span><span class="mi">12345</span><span class="p">)</span>
<span class="n">lr</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">maxIter</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="c1"># We use a ParamGridBuilder to construct a grid of parameters to search over.</span>
<span class="c1"># TrainValidationSplit will try all combinations of values and determine best model using</span>
<span class="c1"># the evaluator.</span>
<span class="n">paramGrid</span> <span class="o">=</span> <span class="n">ParamGridBuilder</span><span class="p">()</span>\
<span class="o">.</span><span class="n">addGrid</span><span class="p">(</span><span class="n">lr</span><span class="o">.</span><span class="n">regParam</span><span class="p">,</span> <span class="p">[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.01</span><span class="p">])</span> \
<span class="o">.</span><span class="n">addGrid</span><span class="p">(</span><span class="n">lr</span><span class="o">.</span><span class="n">fitIntercept</span><span class="p">,</span> <span class="p">[</span><span class="bp">False</span><span class="p">,</span> <span class="bp">True</span><span class="p">])</span>\
<span class="o">.</span><span class="n">addGrid</span><span class="p">(</span><span class="n">lr</span><span class="o">.</span><span class="n">elasticNetParam</span><span class="p">,</span> <span class="p">[</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">])</span>\
<span class="o">.</span><span class="n">build</span><span class="p">()</span>
<span class="c1"># In this case the estimator is simply the linear regression.</span>
<span class="c1"># A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.</span>
<span class="n">tvs</span> <span class="o">=</span> <span class="n">TrainValidationSplit</span><span class="p">(</span><span class="n">estimator</span><span class="o">=</span><span class="n">lr</span><span class="p">,</span>
<span class="n">estimatorParamMaps</span><span class="o">=</span><span class="n">paramGrid</span><span class="p">,</span>
<span class="n">evaluator</span><span class="o">=</span><span class="n">RegressionEvaluator</span><span class="p">(),</span>
<span class="c1"># 80% of the data will be used for training, 20% for validation.</span>
<span class="n">trainRatio</span><span class="o">=</span><span class="mf">0.8</span><span class="p">)</span>
<span class="c1"># Run TrainValidationSplit, and choose the best set of parameters.</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">tvs</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">train</span><span class="p">)</span>
<span class="c1"># Make predictions on test data. model is the model with combination of parameters</span>
<span class="c1"># that performed best.</span>
<span class="n">model</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">test</span><span class="p">)</span>\
<span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;features&quot;</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">,</span> <span class="s2">&quot;prediction&quot;</span><span class="p">)</span>\
<span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
<div><small>Find full example code at "examples/src/main/python/ml/train_validation_split.py" in the Spark repo.</small></div>
</div>
</div>
</div>
<!-- /container -->
</div>
<script src="js/vendor/jquery-1.8.0.min.js"></script>
<script src="js/vendor/bootstrap.min.js"></script>
<script src="js/vendor/anchor.min.js"></script>
<script src="js/main.js"></script>
<!-- MathJax Section -->
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: { equationNumbers: { autoNumber: "AMS" } }
});
</script>
<script>
// Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS.
// We could use "//cdn.mathjax...", but that won't support "file://".
(function(d, script) {
script = d.createElement('script');
script.type = 'text/javascript';
script.async = true;
script.onload = function(){
MathJax.Hub.Config({
tex2jax: {
inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ],
displayMath: [ ["$$","$$"], ["\\[", "\\]"] ],
processEscapes: true,
skipTags: ['script', 'noscript', 'style', 'textarea', 'pre']
}
});
};
script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
'cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML';
d.getElementsByTagName('head')[0].appendChild(script);
}(document));
</script>
</body>
</html>