content/docs/0.9/libs/ml/index.html - flink-web - Git at Google

 <!--
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
 regarding copyright ownership.  The ASF licenses this file
 to you under the Apache License, Version 2.0 (the
 "License"); you may not use this file except in compliance
 with the License.  You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing,
 software distributed under the License is distributed on an
 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 -->
 <!DOCTYPE html>

 <html lang="en">
   <head>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->

     <title>Apache Flink 0.9.0 Documentation: FlinkML - Machine Learning for Flink</title>

     <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
     <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">

     <!-- Bootstrap -->
     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
     <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
     <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
     <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">

     <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
     <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
     <!--[if lt IE 9]>
       <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
       <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
     <![endif]-->
   </head>
   <body>


     <!-- Top navbar. -->
     <nav class="navbar navbar-default navbar-fixed-top">
       <div class="container">
         <!-- The logo. -->
         <div class="navbar-header">
           <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
           </button>
           <div class="navbar-logo">
             <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
           </div>
         </div><!-- /.navbar-header -->

         <!-- The navigation links. -->
         <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
           <ul class="nav navbar-nav">
             <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>

             <!-- Setup -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
               <ul class="dropdown-menu" role="menu">
                 <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>

                 <li class="divider"></li>
                 <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
                 <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>

                 <li class="divider"></li>
                 <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
               </ul>
             </li>

             <!-- Programming Guides -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
               <ul class="dropdown-menu" role="menu">
                 <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>

                 <li class="divider"></li>
                 <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
               </ul>
             </li>

             <!-- Libraries -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
                 <ul class="dropdown-menu" role="menu">
                   <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
                   <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
                   <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
                   <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
               </ul>
             </li>

             <!-- Internals -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
               <ul class="dropdown-menu" role="menu">
                 <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
                 <li class="divider"></li>
                 <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture &amp; Process Model</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction &amp; Serialization</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs &amp; Scheduling</a></li>
                 <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
               </ul>
             </li>
           </ul>
           <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
             <div class="form-group">
               <input type="text" class="form-control" name="q" placeholder="Search all pages">
             </div>
             <button type="submit" class="btn btn-default">Search</button>
           </form>
         </div><!-- /.navbar-collapse -->
       </div><!-- /.container -->
     </nav>


     <!-- Main content. -->
     <div class="container">


 <div class="row">
   <div class="col-sm-10 col-sm-offset-1">
     <h1>FlinkML - Machine Learning for Flink</h1>


 <p>FlinkML is the Machine Learning (ML) library for Flink. It is a new effort in the Flink community,
 with a growing list of algorithms and contributors. With FlinkML we aim to provide
 scalable ML algorithms, an intuitive API, and tools that help minimize glue code in end-to-end ML
 systems. You can see more details about our goals and where the library is headed in our <a href="vision_roadmap.html">vision
 and roadmap here</a>.</p>

 <ul id="markdown-toc">
   <li><a href="#supported-algorithms" id="markdown-toc-supported-algorithms">Supported Algorithms</a>    <ul>
       <li><a href="#supervised-learning" id="markdown-toc-supervised-learning">Supervised Learning</a></li>
       <li><a href="#data-preprocessing" id="markdown-toc-data-preprocessing">Data Preprocessing</a></li>
       <li><a href="#recommendation" id="markdown-toc-recommendation">Recommendation</a></li>
       <li><a href="#utilities" id="markdown-toc-utilities">Utilities</a></li>
     </ul>
   </li>
   <li><a href="#getting-started" id="markdown-toc-getting-started">Getting Started</a></li>
   <li><a href="#pipelines" id="markdown-toc-pipelines">Pipelines</a></li>
   <li><a href="#how-to-contribute" id="markdown-toc-how-to-contribute">How to contribute</a></li>
 </ul>

 <h2 id="supported-algorithms">Supported Algorithms</h2>

 <p>FlinkML currently supports the following algorithms:</p>

 <h3 id="supervised-learning">Supervised Learning</h3>

 <ul>
   <li><a href="svm.html">SVM using Communication efficient distributed dual coordinate ascent (CoCoA)</a></li>
   <li><a href="multiple_linear_regression.html">Multiple linear regression</a></li>
   <li><a href="optimization.html">Optimization Framework</a></li>
 </ul>

 <h3 id="data-preprocessing">Data Preprocessing</h3>

 <ul>
   <li><a href="polynomial_features.html">Polynomial Features</a></li>
   <li><a href="standard_scaler.html">Standard Scaler</a></li>
   <li><a href="min_max_scaler.html">MinMax Scaler</a></li>
 </ul>

 <h3 id="recommendation">Recommendation</h3>

 <ul>
   <li><a href="als.html">Alternating Least Squares (ALS)</a></li>
 </ul>

 <h3 id="utilities">Utilities</h3>

 <ul>
   <li><a href="distance_metrics.html">Distance Metrics</a></li>
 </ul>

 <h2 id="getting-started">Getting Started</h2>

 <p>You can check out our <a href="quickstart.html">quickstart guide</a> for a comprehensive getting started
 example.</p>

 <p>If you want to jump right in, you have to <a href="http://ci.apache.org/projects/flink/flink-docs-master/apis/programming_guide.html#linking-with-flink">set up a Flink program</a>.
 Next, you have to add the FlinkML dependency to the <code>pom.xml</code> of your project.</p>

 <div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt">&lt;dependency&gt;</span>
   <span class="nt">&lt;groupId&gt;</span>org.apache.flink<span class="nt">&lt;/groupId&gt;</span>
   <span class="nt">&lt;artifactId&gt;</span>flink-ml<span class="nt">&lt;/artifactId&gt;</span>
   <span class="nt">&lt;version&gt;</span>0.9.0<span class="nt">&lt;/version&gt;</span>
 <span class="nt">&lt;/dependency&gt;</span></code></pre></div>

 <p>Note that FlinkML is currently not part of the binary distribution. See linking with it for cluster execution <a href="../apis/cluster_execution.html#linking-with-modules-not-contained-in-the-binary-distribution">here</a>.</p>

 <p>Now you can start solving your analysis task.
 The following code snippet shows how easy it is to train a multiple linear regression model.</p>

 <div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="c1">// LabeledVector is a feature vector with a label (class or real value)</span>
 <span class="k">val</span> <span class="n">trainingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>
 <span class="k">val</span> <span class="n">testingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>

 <span class="k">val</span> <span class="n">mlr</span> <span class="k">=</span> <span class="nc">MultipleLinearRegression</span><span class="o">()</span>
   <span class="o">.</span><span class="n">setStepsize</span><span class="o">(</span><span class="mf">1.0</span><span class="o">)</span>
   <span class="o">.</span><span class="n">setIterations</span><span class="o">(</span><span class="mi">100</span><span class="o">)</span>
   <span class="o">.</span><span class="n">setConvergenceThreshold</span><span class="o">(</span><span class="mf">0.001</span><span class="o">)</span>

 <span class="n">mlr</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">trainingData</span><span class="o">,</span> <span class="n">parameters</span><span class="o">)</span>

 <span class="c1">// The fitted model can now be used to make predictions</span>
 <span class="k">val</span> <span class="n">predictions</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">mlr</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">testingData</span><span class="o">)</span></code></pre></div>

 <h2 id="pipelines">Pipelines</h2>

 <p>A key concept of FlinkML is its <a href="http://scikit-learn.org">scikit-learn</a> inspired pipelining mechanism.
 It allows you to quickly build complex data analysis pipelines how they appear in every data scientist’s daily work.
 An in-depth description of FlinkML’s pipelines and their internal workings can be found <a href="pipelines.html">here</a>.</p>

 <p>The following example code shows how easy it is to set up an analysis pipeline with FlinkML.</p>

 <div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">val</span> <span class="n">trainingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>
 <span class="k">val</span> <span class="n">testingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>

 <span class="k">val</span> <span class="n">scaler</span> <span class="k">=</span> <span class="nc">StandardScaler</span><span class="o">()</span>
 <span class="k">val</span> <span class="n">polyFeatures</span> <span class="k">=</span> <span class="nc">PolynomialFeatures</span><span class="o">().</span><span class="n">setDegree</span><span class="o">(</span><span class="mi">3</span><span class="o">)</span>
 <span class="k">val</span> <span class="n">mlr</span> <span class="k">=</span> <span class="nc">MultipleLinearRegression</span><span class="o">()</span>

 <span class="c1">// Construct pipeline of standard scaler, polynomial features and multiple linear regression</span>
 <span class="k">val</span> <span class="n">pipeline</span> <span class="k">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">chainTransformer</span><span class="o">(</span><span class="n">polyFeatures</span><span class="o">).</span><span class="n">chainPredictor</span><span class="o">(</span><span class="n">mlr</span><span class="o">)</span>

 <span class="c1">// Train pipeline</span>
 <span class="n">pipeline</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">trainingData</span><span class="o">)</span>

 <span class="c1">// Calculate predictions</span>
 <span class="k">val</span> <span class="n">predictions</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">pipeline</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">testingData</span><span class="o">)</span></code></pre></div>

 <p>One can chain a <code>Transformer</code> to another <code>Transformer</code> or a set of chained <code>Transformers</code> by calling the method <code>chainTransformer</code>.
 If one wants to chain a <code>Predictor</code> to a <code>Transformer</code> or a set of chained <code>Transformers</code>, one has to call the method <code>chainPredictor</code>.</p>

 <h2 id="how-to-contribute">How to contribute</h2>

 <p>The Flink community welcomes all contributors who want to get involved in the development of Flink and its libraries.
 In order to get quickly started with contributing to FlinkML, please read our official
 <a href="http://flink.apache.org/docs/0.9/libs/ml/contribution_guide.html">contribution guide</a>.</p>

   </div>

   <div class="col-sm-10 col-sm-offset-1">
     <!-- Disqus thread and some vertical offset -->
     <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
   </div>
 </div>

     </div><!-- /.container -->

     <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
     <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
     <!-- Include all compiled plugins (below), or include individual files as needed -->
     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
     <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>

     <!-- Google Analytics -->
     <script>
       (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
       m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
       })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

       ga('create', 'UA-52545728-1', 'auto');
       ga('send', 'pageview');
     </script>

     <!-- Disqus -->
     <script type="text/javascript">
     var disqus_shortname = 'stratosphere-eu';
     (function() {
         var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
         dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
         (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
     })();
 </script>
   </body>
 </html>
	<!--
	Licensed to the Apache Software Foundation (ASF) under one
	or more contributor license agreements. See the NOTICE file
	distributed with this work for additional information
	regarding copyright ownership. The ASF licenses this file
	to you under the Apache License, Version 2.0 (the
	"License"); you may not use this file except in compliance
	with the License. You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing,
	software distributed under the License is distributed on an
	"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	KIND, either express or implied. See the License for the
	specific language governing permissions and limitations
	under the License.
	-->
	<!DOCTYPE html>

	<html lang="en">
	<head>
	<meta charset="utf-8">
	<meta http-equiv="X-UA-Compatible" content="IE=edge">
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<!-- The above 3 meta tags must come first in the head; any other head content must come after these tags -->

	<title>Apache Flink 0.9.0 Documentation: FlinkML - Machine Learning for Flink</title>

	<link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
	<link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">

	<!-- Bootstrap -->
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
	<link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
	<link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
	<link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">

	<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
	<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
	<!--[if lt IE 9]>
	<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
	<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
	<![endif]-->
	</head>
	<body>






	<!-- Top navbar. -->
	<nav class="navbar navbar-default navbar-fixed-top">
	<div class="container">
	<!-- The logo. -->
	<div class="navbar-header">
	<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	</button>
	<div class="navbar-logo">
	<a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
	</div>
	</div><!-- /.navbar-header -->

	<!-- The navigation links. -->
	<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
	<ul class="nav navbar-nav">
	<li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>

	<!-- Setup -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>

	<li class="divider"></li>
	<li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
	<li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>

	<li class="divider"></li>
	<li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
	</ul>
	</li>

	<!-- Programming Guides -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>

	<li class="divider"></li>
	<li><a href="scala_shell.html">Interactive Scala Shell</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
	</ul>
	</li>

	<!-- Libraries -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
	<li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
	<li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
	</ul>
	</li>

	<!-- Internals -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
	<li class="divider"></li>
	<li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture & Process Model</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction & Serialization</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs & Scheduling</a></li>
	<li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
	</ul>
	</li>
	</ul>
	<form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
	<div class="form-group">
	<input type="text" class="form-control" name="q" placeholder="Search all pages">
	</div>
	<button type="submit" class="btn btn-default">Search</button>
	</form>
	</div><!-- /.navbar-collapse -->
	</div><!-- /.container -->
	</nav>




	<!-- Main content. -->
	<div class="container">


	<div class="row">
	<div class="col-sm-10 col-sm-offset-1">
	<h1>FlinkML - Machine Learning for Flink</h1>



	<p>FlinkML is the Machine Learning (ML) library for Flink. It is a new effort in the Flink community,
	with a growing list of algorithms and contributors. With FlinkML we aim to provide
	scalable ML algorithms, an intuitive API, and tools that help minimize glue code in end-to-end ML
	systems. You can see more details about our goals and where the library is headed in our <a href="vision_roadmap.html">vision
	and roadmap here</a>.</p>

	<ul id="markdown-toc">
	<li><a href="#supported-algorithms" id="markdown-toc-supported-algorithms">Supported Algorithms</a> <ul>
	<li><a href="#supervised-learning" id="markdown-toc-supervised-learning">Supervised Learning</a></li>
	<li><a href="#data-preprocessing" id="markdown-toc-data-preprocessing">Data Preprocessing</a></li>
	<li><a href="#recommendation" id="markdown-toc-recommendation">Recommendation</a></li>
	<li><a href="#utilities" id="markdown-toc-utilities">Utilities</a></li>
	</ul>
	</li>
	<li><a href="#getting-started" id="markdown-toc-getting-started">Getting Started</a></li>
	<li><a href="#pipelines" id="markdown-toc-pipelines">Pipelines</a></li>
	<li><a href="#how-to-contribute" id="markdown-toc-how-to-contribute">How to contribute</a></li>
	</ul>

	<h2 id="supported-algorithms">Supported Algorithms</h2>

	<p>FlinkML currently supports the following algorithms:</p>

	<h3 id="supervised-learning">Supervised Learning</h3>

	<ul>
	<li><a href="svm.html">SVM using Communication efficient distributed dual coordinate ascent (CoCoA)</a></li>
	<li><a href="multiple_linear_regression.html">Multiple linear regression</a></li>
	<li><a href="optimization.html">Optimization Framework</a></li>
	</ul>

	<h3 id="data-preprocessing">Data Preprocessing</h3>

	<ul>
	<li><a href="polynomial_features.html">Polynomial Features</a></li>
	<li><a href="standard_scaler.html">Standard Scaler</a></li>
	<li><a href="min_max_scaler.html">MinMax Scaler</a></li>
	</ul>

	<h3 id="recommendation">Recommendation</h3>

	<ul>
	<li><a href="als.html">Alternating Least Squares (ALS)</a></li>
	</ul>

	<h3 id="utilities">Utilities</h3>

	<ul>
	<li><a href="distance_metrics.html">Distance Metrics</a></li>
	</ul>

	<h2 id="getting-started">Getting Started</h2>

	<p>You can check out our <a href="quickstart.html">quickstart guide</a> for a comprehensive getting started
	example.</p>

	<p>If you want to jump right in, you have to <a href="http://ci.apache.org/projects/flink/flink-docs-master/apis/programming_guide.html#linking-with-flink">set up a Flink program</a>.
	Next, you have to add the FlinkML dependency to the <code>pom.xml</code> of your project.</p>

	<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><dependency></span>
	<span class="nt"><groupId></span>org.apache.flink<span class="nt"></groupId></span>
	<span class="nt"><artifactId></span>flink-ml<span class="nt"></artifactId></span>
	<span class="nt"><version></span>0.9.0<span class="nt"></version></span>
	<span class="nt"></dependency></span></code></pre></div>

	<p>Note that FlinkML is currently not part of the binary distribution. See linking with it for cluster execution <a href="../apis/cluster_execution.html#linking-with-modules-not-contained-in-the-binary-distribution">here</a>.</p>

	<p>Now you can start solving your analysis task.
	The following code snippet shows how easy it is to train a multiple linear regression model.</p>

	<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="c1">// LabeledVector is a feature vector with a label (class or real value)</span>
	<span class="k">val</span> <span class="n">trainingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>
	<span class="k">val</span> <span class="n">testingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>

	<span class="k">val</span> <span class="n">mlr</span> <span class="k">=</span> <span class="nc">MultipleLinearRegression</span><span class="o">()</span>
	<span class="o">.</span><span class="n">setStepsize</span><span class="o">(</span><span class="mf">1.0</span><span class="o">)</span>
	<span class="o">.</span><span class="n">setIterations</span><span class="o">(</span><span class="mi">100</span><span class="o">)</span>
	<span class="o">.</span><span class="n">setConvergenceThreshold</span><span class="o">(</span><span class="mf">0.001</span><span class="o">)</span>

	<span class="n">mlr</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">trainingData</span><span class="o">,</span> <span class="n">parameters</span><span class="o">)</span>

	<span class="c1">// The fitted model can now be used to make predictions</span>
	<span class="k">val</span> <span class="n">predictions</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">mlr</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">testingData</span><span class="o">)</span></code></pre></div>

	<h2 id="pipelines">Pipelines</h2>

	<p>A key concept of FlinkML is its <a href="http://scikit-learn.org">scikit-learn</a> inspired pipelining mechanism.
	It allows you to quickly build complex data analysis pipelines how they appear in every data scientist’s daily work.
	An in-depth description of FlinkML’s pipelines and their internal workings can be found <a href="pipelines.html">here</a>.</p>

	<p>The following example code shows how easy it is to set up an analysis pipeline with FlinkML.</p>

	<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">val</span> <span class="n">trainingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>
	<span class="k">val</span> <span class="n">testingData</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>

	<span class="k">val</span> <span class="n">scaler</span> <span class="k">=</span> <span class="nc">StandardScaler</span><span class="o">()</span>
	<span class="k">val</span> <span class="n">polyFeatures</span> <span class="k">=</span> <span class="nc">PolynomialFeatures</span><span class="o">().</span><span class="n">setDegree</span><span class="o">(</span><span class="mi">3</span><span class="o">)</span>
	<span class="k">val</span> <span class="n">mlr</span> <span class="k">=</span> <span class="nc">MultipleLinearRegression</span><span class="o">()</span>

	<span class="c1">// Construct pipeline of standard scaler, polynomial features and multiple linear regression</span>
	<span class="k">val</span> <span class="n">pipeline</span> <span class="k">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">chainTransformer</span><span class="o">(</span><span class="n">polyFeatures</span><span class="o">).</span><span class="n">chainPredictor</span><span class="o">(</span><span class="n">mlr</span><span class="o">)</span>

	<span class="c1">// Train pipeline</span>
	<span class="n">pipeline</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">trainingData</span><span class="o">)</span>

	<span class="c1">// Calculate predictions</span>
	<span class="k">val</span> <span class="n">predictions</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">pipeline</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">testingData</span><span class="o">)</span></code></pre></div>

	<p>One can chain a <code>Transformer</code> to another <code>Transformer</code> or a set of chained <code>Transformers</code> by calling the method <code>chainTransformer</code>.
	If one wants to chain a <code>Predictor</code> to a <code>Transformer</code> or a set of chained <code>Transformers</code>, one has to call the method <code>chainPredictor</code>.</p>

	<h2 id="how-to-contribute">How to contribute</h2>

	<p>The Flink community welcomes all contributors who want to get involved in the development of Flink and its libraries.
	In order to get quickly started with contributing to FlinkML, please read our official
	<a href="http://flink.apache.org/docs/0.9/libs/ml/contribution_guide.html">contribution guide</a>.</p>

	</div>

	<div class="col-sm-10 col-sm-offset-1">
	<!-- Disqus thread and some vertical offset -->
	<div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
	</div>
	</div>

	</div><!-- /.container -->

	<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
	<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
	<!-- Include all compiled plugins (below), or include individual files as needed -->
	<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
	<script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>

	<!-- Google Analytics -->
	<script>
	(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]\|\|function(){
	(i[r].q=i[r].q\|\|[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','//www.google-analytics.com/analytics.js','ga');

	ga('create', 'UA-52545728-1', 'auto');
	ga('send', 'pageview');
	</script>

	<!-- Disqus -->
	<script type="text/javascript">
	var disqus_shortname = 'stratosphere-eu';
	(function() {
	var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
	dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
	(document.getElementsByTagName('head')[0] \|\| document.getElementsByTagName('body')[0]).appendChild(dsq);
	})();
	</script>
	</body>
	</html>