docs/master/quickstart/scala_api_quickstart.html - flink-web - Git at Google

 <!--
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
 regarding copyright ownership.  The ASF licenses this file
 to you under the Apache License, Version 2.0 (the
 "License"); you may not use this file except in compliance
 with the License.  You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing,
 software distributed under the License is distributed on an
 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 KIND, either express or implied.  See the License for the
 specific language governing permissions and limitations
 under the License.
 -->
 <!DOCTYPE html>

 <html lang="en">
   <head>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->

     <title>Apache Flink 0.10-SNAPSHOT Documentation: Quickstart: Scala API</title>

     <link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico" type="image/x-icon">
     <link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico" type="image/x-icon">

     <!-- Bootstrap -->
     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
     <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css">
     <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css">
     <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css">

     <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
     <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
     <!--[if lt IE 9]>
       <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
       <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
     <![endif]-->
   </head>
   <body>


     <!-- Top navbar. -->
     <nav class="navbar navbar-default navbar-fixed-top">
       <div class="container">
         <!-- The logo. -->
         <div class="navbar-header">
           <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
           </button>
           <div class="navbar-logo">
             <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg"></a>
           </div>
         </div><!-- /.navbar-header -->

         <!-- The navigation links. -->
         <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
           <ul class="nav navbar-nav">
             <li><a href="http://flink.apache.org/docs/master/index.html">Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li>

             <!-- Setup -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/master/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
               <ul class="dropdown-menu" role="menu">
                 <li><a href="http://flink.apache.org/docs/master/setup/building.html">Get Flink 0.10-SNAPSHOT</a></li>

                 <li class="divider"></li>
                 <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
                 <li><a href="http://flink.apache.org/docs/master/setup/local_setup.html" class="active">Local</a></li>
                 <li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html">Cluster (Standalone)</a></li>
                 <li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html">YARN</a></li>
                 <li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html">GCloud</a></li>
                 <li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>

                 <li class="divider"></li>
                 <li><a href="http://flink.apache.org/docs/master/setup/config.html">Configuration</a></li>
               </ul>
             </li>

             <!-- Programming Guides -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/master/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
               <ul class="dropdown-menu" role="menu">
                 <li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/python.html">Python API <span class="badge">Beta</span></a></li>

                 <li class="divider"></li>
                 <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html">Dataset Transformations</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/best_practices.html">Best Practices</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html">Connectors</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/examples.html">Examples</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/local_execution.html">Local Execution</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html">Cluster Execution</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/cli.html">Command Line Interface</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/web_client.html">Web Client</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/iterations.html">Iterations</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/java8.html">Java 8</a></li>
                 <li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
               </ul>
             </li>

             <!-- Libraries -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/master/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
                 <ul class="dropdown-menu" role="menu">
                   <li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html">Graphs: Spargel</a></li>
                   <li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
                   <li><a href="http://flink.apache.org/docs/master/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
                   <li><a href="http://flink.apache.org/docs/master/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
               </ul>
             </li>

             <!-- Internals -->
             <li class="dropdown">
               <a href="http://flink.apache.org/docs/master/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
               <ul class="dropdown-menu" role="menu">
                 <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html">How to Contribute</a></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html">Coding Guidelines</a></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html">IDE Setup</a></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/logging.html">Logging</a></li>
                 <li class="divider"></li>
                 <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/general_arch.html">Architecture &amp; Process Model</a></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html">Type Extraction &amp; Serialization</a></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html">Jobs &amp; Scheduling</a></li>
                 <li><a href="http://flink.apache.org/docs/master/internals/add_operator.html">How-To: Add an Operator</a></li>
               </ul>
             </li>
           </ul>
           <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html">
             <div class="form-group">
               <input type="text" class="form-control" name="q" placeholder="Search all pages">
             </div>
             <button type="submit" class="btn btn-default">Search</button>
           </form>
         </div><!-- /.navbar-collapse -->
       </div><!-- /.container -->
     </nav>


     <!-- Main content. -->
     <div class="container">


 <div class="row">
   <div class="col-sm-10 col-sm-offset-1">
     <h1>Quickstart: Scala API</h1>


 <ul id="markdown-toc">
   <li><a href="#requirements" id="markdown-toc-requirements">Requirements</a></li>
   <li><a href="#create-project" id="markdown-toc-create-project">Create Project</a></li>
   <li><a href="#inspect-project" id="markdown-toc-inspect-project">Inspect Project</a></li>
   <li><a href="#build-project" id="markdown-toc-build-project">Build Project</a></li>
   <li><a href="#next-steps" id="markdown-toc-next-steps">Next Steps</a></li>
   <li><a href="#alternative-build-tools-sbt" id="markdown-toc-alternative-build-tools-sbt">Alternative Build Tools: SBT</a></li>
 </ul>

 <p>Start working on your Flink Scala program in a few simple steps.</p>

 <h2 id="requirements">Requirements</h2>

 <p>The only requirements are working <strong>Maven 3.0.4</strong> (or higher) and <strong>Java 6.x</strong> (or higher) installations.</p>

 <h2 id="create-project">Create Project</h2>

 <p>Use one of the following commands to <strong>create a project</strong>:</p>

 <ul class="nav nav-tabs" style="border-bottom: none;">
     <li class="active"><a href="#quickstart-script" data-toggle="tab">Run the <strong>quickstart script</strong></a></li>
     <li><a href="#maven-archetype" data-toggle="tab">Use <strong>Maven archetypes</strong></a></li>
 </ul>
 <div class="tab-content">
     <div class="tab-pane active" id="quickstart-script">

 <div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>curl http://flink.apache.org/q/quickstart-scala.sh <span class="p">|</span> bash</code></pre></div>

     </div>
     <div class="tab-pane" id="maven-archetype">

 <div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>mvn archetype:generate                             <span class="se">\</span>
   -DarchetypeGroupId<span class="o">=</span>org.apache.flink              <span class="se">\</span>
   -DarchetypeArtifactId<span class="o">=</span>flink-quickstart-scala           <span class="se">\</span>
   -DarchetypeVersion<span class="o">=</span>0.10-SNAPSHOT</code></pre></div>

     This allows you to <strong>name your newly created project</strong>. It will interactively ask you for the groupId, artifactId, and package name.
     </div>
 </div>

 <h2 id="inspect-project">Inspect Project</h2>

 <p>There will be a new directory in your working directory. If you’ve used the <em>curl</em> approach, the directory is called <code>quickstart</code>. Otherwise, it has the name of your artifactId.</p>

 <p>The sample project is a <strong>Maven project</strong>, which contains two classes. <em>Job</em> is a basic skeleton program and <em>WordCountJob</em> a working example. Please note that the <em>main</em> method of both classes allow you to start Flink in a development/testing mode.</p>

 <p>We recommend to <strong>import this project into your IDE</strong>. For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites:</p>

 <ul>
   <li><em>Eclipse 4.x</em>
     <ul>
       <li><a href="http://download.scala-ide.org/sdk/e38/scala210/stable/site">Scala IDE</a></li>
       <li><a href="http://alchim31.free.fr/m2e-scala/update-site">m2eclipse-scala</a></li>
       <li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.15.0/N/0.15.0.201206251206/">Build Helper Maven Plugin</a></li>
     </ul>
   </li>
   <li><em>Eclipse 3.7</em>
     <ul>
       <li><a href="http://download.scala-ide.org/sdk/e37/scala210/stable/site">Scala IDE</a></li>
       <li><a href="http://alchim31.free.fr/m2e-scala/update-site">m2eclipse-scala</a></li>
       <li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/">Build Helper Maven Plugin</a></li>
     </ul>
   </li>
 </ul>

 <p>The IntelliJ IDE also supports Maven and offers a plugin for Scala development.</p>

 <h2 id="build-project">Build Project</h2>

 <p>If you want to <strong>build your project</strong>, go to your project directory and issue the <code>mvn clean package -Pbuild-jar</code> command. You will <strong>find a jar</strong> that runs on every Flink cluster in <strong>target/your-artifact-id-1.0-SNAPSHOT.jar</strong>. There is also a fat-jar,  <strong>target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar</strong>. This
 also contains all dependencies that get added to the maven project.</p>

 <h2 id="next-steps">Next Steps</h2>

 <p>Write your application!</p>

 <p>The quickstart project contains a WordCount implementation, the “Hello World” of Big Data processing systems. The goal of WordCount is to determine the frequencies of words in a text, e.g., how often do the terms “the” or “house” occurs in all Wikipedia texts.</p>

 <p><strong>Sample Input</strong>:</p>

 <div class="highlight"><pre><code class="language-bash">big data is big</code></pre></div>

 <p><strong>Sample Output</strong>:</p>

 <div class="highlight"><pre><code class="language-bash">big 2
 data 1
 is 1</code></pre></div>

 <p>The following code shows the WordCount implementation from the Quickstart which processes some text lines with two operators (FlatMap and Reduce), and writes the prints the resulting words and counts to std-out.</p>

 <div class="highlight"><pre><code class="language-scala"><span class="k">object</span> <span class="nc">WordCountJob</span> <span class="o">{</span>
   <span class="k">def</span> <span class="n">main</span><span class="o">(</span><span class="n">args</span><span class="k">:</span> <span class="kt">Array</span><span class="o">[</span><span class="kt">String</span><span class="o">])</span> <span class="o">{</span>

     <span class="c1">// set up the execution environment</span>
     <span class="k">val</span> <span class="n">env</span> <span class="k">=</span> <span class="nc">ExecutionEnvironment</span><span class="o">.</span><span class="n">getExecutionEnvironment</span>

     <span class="c1">// get input data</span>
     <span class="k">val</span> <span class="n">text</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">fromElements</span><span class="o">(</span><span class="s">&quot;To be, or not to be,--that is the question:--&quot;</span><span class="o">,</span>
       <span class="s">&quot;Whether &#39;tis nobler in the mind to suffer&quot;</span><span class="o">,</span> <span class="s">&quot;The slings and arrows of outrageous fortune&quot;</span><span class="o">,</span>
       <span class="s">&quot;Or to take arms against a sea of troubles,&quot;</span><span class="o">)</span>

     <span class="k">val</span> <span class="n">counts</span> <span class="k">=</span> <span class="n">text</span><span class="o">.</span><span class="n">flatMap</span> <span class="o">{</span> <span class="k">_</span><span class="o">.</span><span class="n">toLowerCase</span><span class="o">.</span><span class="n">split</span><span class="o">(</span><span class="s">&quot;\\W+&quot;</span><span class="o">)</span> <span class="o">}</span>
       <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="o">(</span><span class="k">_</span><span class="o">,</span> <span class="mi">1</span><span class="o">)</span> <span class="o">}</span>
       <span class="o">.</span><span class="n">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span>
       <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span>

     <span class="c1">// emit result</span>
     <span class="n">counts</span><span class="o">.</span><span class="n">print</span><span class="o">()</span>

     <span class="c1">// execute program</span>
     <span class="n">env</span><span class="o">.</span><span class="n">execute</span><span class="o">(</span><span class="s">&quot;WordCount Example&quot;</span><span class="o">)</span>
   <span class="o">}</span>
 <span class="o">}</span></code></pre></div>

 <p><a href="https://github.com/apache/flink/blob/master//flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala">Check GitHub</a> for the full example code.</p>

 <p>For a complete overview over our API, have a look at the <a href="http://flink.apache.org/docs/master/apis/programming_guide.html">Programming Guide</a> and <a href="http://flink.apache.org/docs/master/apis/examples.html">further example programs</a>. If you have any trouble, ask on our <a href="http://mail-archives.apache.org/mod_mbox/flink-dev/">Mailing List</a>. We are happy to provide help.</p>

 <h2 id="alternative-build-tools-sbt">Alternative Build Tools: SBT</h2>

 <p>To build and run applications with SBT instead of Maven is pretty straight forward. After creating the standard sbt <a href="http://www.scala-sbt.org/0.13/tutorial/Directories.html">directory layout</a> it’s enough to add the Flink dependencies to the <code>build.sbt</code> file:</p>

 <div class="highlight"><pre><code class="language-scala"><span class="n">libraryDependencies</span> <span class="o">++=</span> <span class="nc">Seq</span><span class="o">(</span><span class="s">&quot;org.apache.flink&quot;</span> <span class="o">%</span> <span class="s">&quot;flink-scala&quot;</span> <span class="o">%</span> <span class="s">&quot;0.10-SNAPSHOT&quot;</span><span class="o">,</span> <span class="s">&quot;org.apache.flink&quot;</span> <span class="o">%</span> <span class="s">&quot;flink-clients&quot;</span> <span class="o">%</span> <span class="s">&quot;0.10-SNAPSHOT&quot;</span><span class="o">)</span></code></pre></div>

 <p>Now the application can be executed by <code>sbt run</code>. By default SBT runs an application in the same JVM itself is running in. This can lead to lass loading issues with Flink. To avoid these, append the following line to <code>build.sbt</code>:</p>

 <div class="highlight"><pre><code class="language-scala"><span class="n">fork</span> <span class="n">in</span> <span class="n">run</span> <span class="o">:=</span> <span class="kc">true</span></code></pre></div>


   </div>

   <div class="col-sm-10 col-sm-offset-1">
     <!-- Disqus thread and some vertical offset -->
     <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
   </div>
 </div>

     </div><!-- /.container -->

     <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
     <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
     <!-- Include all compiled plugins (below), or include individual files as needed -->
     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
     <script src="http://flink.apache.org/docs/master/page/js/codetabs.js"></script>

     <!-- Google Analytics -->
     <script>
       (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
       m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
       })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

       ga('create', 'UA-52545728-1', 'auto');
       ga('send', 'pageview');
     </script>

     <!-- Disqus -->
     <script type="text/javascript">
     var disqus_shortname = 'stratosphere-eu';
     (function() {
         var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
         dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
         (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
     })();
 </script>
   </body>
 </html>
	<!--
	Licensed to the Apache Software Foundation (ASF) under one
	or more contributor license agreements. See the NOTICE file
	distributed with this work for additional information
	regarding copyright ownership. The ASF licenses this file
	to you under the Apache License, Version 2.0 (the
	"License"); you may not use this file except in compliance
	with the License. You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing,
	software distributed under the License is distributed on an
	"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	KIND, either express or implied. See the License for the
	specific language governing permissions and limitations
	under the License.
	-->
	<!DOCTYPE html>

	<html lang="en">
	<head>
	<meta charset="utf-8">
	<meta http-equiv="X-UA-Compatible" content="IE=edge">
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<!-- The above 3 meta tags must come first in the head; any other head content must come after these tags -->

	<title>Apache Flink 0.10-SNAPSHOT Documentation: Quickstart: Scala API</title>

	<link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico" type="image/x-icon">
	<link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico" type="image/x-icon">

	<!-- Bootstrap -->
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
	<link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css">
	<link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css">
	<link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css">

	<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
	<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
	<!--[if lt IE 9]>
	<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
	<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
	<![endif]-->
	</head>
	<body>






	<!-- Top navbar. -->
	<nav class="navbar navbar-default navbar-fixed-top">
	<div class="container">
	<!-- The logo. -->
	<div class="navbar-header">
	<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	</button>
	<div class="navbar-logo">
	<a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg"></a>
	</div>
	</div><!-- /.navbar-header -->

	<!-- The navigation links. -->
	<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
	<ul class="nav navbar-nav">
	<li><a href="http://flink.apache.org/docs/master/index.html">Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li>

	<!-- Setup -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/master/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li><a href="http://flink.apache.org/docs/master/setup/building.html">Get Flink 0.10-SNAPSHOT</a></li>

	<li class="divider"></li>
	<li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
	<li><a href="http://flink.apache.org/docs/master/setup/local_setup.html" class="active">Local</a></li>
	<li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html">Cluster (Standalone)</a></li>
	<li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html">YARN</a></li>
	<li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html">GCloud</a></li>
	<li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>

	<li class="divider"></li>
	<li><a href="http://flink.apache.org/docs/master/setup/config.html">Configuration</a></li>
	</ul>
	</li>

	<!-- Programming Guides -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/master/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/python.html">Python API <span class="badge">Beta</span></a></li>

	<li class="divider"></li>
	<li><a href="scala_shell.html">Interactive Scala Shell</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html">Dataset Transformations</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/best_practices.html">Best Practices</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html">Connectors</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/examples.html">Examples</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/local_execution.html">Local Execution</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html">Cluster Execution</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/cli.html">Command Line Interface</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/web_client.html">Web Client</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/iterations.html">Iterations</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/java8.html">Java 8</a></li>
	<li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
	</ul>
	</li>

	<!-- Libraries -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/master/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html">Graphs: Spargel</a></li>
	<li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
	<li><a href="http://flink.apache.org/docs/master/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
	<li><a href="http://flink.apache.org/docs/master/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
	</ul>
	</li>

	<!-- Internals -->
	<li class="dropdown">
	<a href="http://flink.apache.org/docs/master/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
	<ul class="dropdown-menu" role="menu">
	<li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
	<li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html">How to Contribute</a></li>
	<li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html">Coding Guidelines</a></li>
	<li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html">IDE Setup</a></li>
	<li><a href="http://flink.apache.org/docs/master/internals/logging.html">Logging</a></li>
	<li class="divider"></li>
	<li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
	<li><a href="http://flink.apache.org/docs/master/internals/general_arch.html">Architecture & Process Model</a></li>
	<li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html">Type Extraction & Serialization</a></li>
	<li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html">Jobs & Scheduling</a></li>
	<li><a href="http://flink.apache.org/docs/master/internals/add_operator.html">How-To: Add an Operator</a></li>
	</ul>
	</li>
	</ul>
	<form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html">
	<div class="form-group">
	<input type="text" class="form-control" name="q" placeholder="Search all pages">
	</div>
	<button type="submit" class="btn btn-default">Search</button>
	</form>
	</div><!-- /.navbar-collapse -->
	</div><!-- /.container -->
	</nav>




	<!-- Main content. -->
	<div class="container">


	<div class="row">
	<div class="col-sm-10 col-sm-offset-1">
	<h1>Quickstart: Scala API</h1>



	<ul id="markdown-toc">
	<li><a href="#requirements" id="markdown-toc-requirements">Requirements</a></li>
	<li><a href="#create-project" id="markdown-toc-create-project">Create Project</a></li>
	<li><a href="#inspect-project" id="markdown-toc-inspect-project">Inspect Project</a></li>
	<li><a href="#build-project" id="markdown-toc-build-project">Build Project</a></li>
	<li><a href="#next-steps" id="markdown-toc-next-steps">Next Steps</a></li>
	<li><a href="#alternative-build-tools-sbt" id="markdown-toc-alternative-build-tools-sbt">Alternative Build Tools: SBT</a></li>
	</ul>

	<p>Start working on your Flink Scala program in a few simple steps.</p>

	<h2 id="requirements">Requirements</h2>

	<p>The only requirements are working <strong>Maven 3.0.4</strong> (or higher) and <strong>Java 6.x</strong> (or higher) installations.</p>

	<h2 id="create-project">Create Project</h2>

	<p>Use one of the following commands to <strong>create a project</strong>:</p>

	<ul class="nav nav-tabs" style="border-bottom: none;">
	<li class="active"><a href="#quickstart-script" data-toggle="tab">Run the <strong>quickstart script</strong></a></li>
	<li><a href="#maven-archetype" data-toggle="tab">Use <strong>Maven archetypes</strong></a></li>
	</ul>
	<div class="tab-content">
	<div class="tab-pane active" id="quickstart-script">

	<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>curl http://flink.apache.org/q/quickstart-scala.sh <span class="p">\|</span> bash</code></pre></div>

	</div>
	<div class="tab-pane" id="maven-archetype">

	<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>mvn archetype:generate <span class="se">\</span>
	-DarchetypeGroupId<span class="o">=</span>org.apache.flink <span class="se">\</span>
	-DarchetypeArtifactId<span class="o">=</span>flink-quickstart-scala <span class="se">\</span>
	-DarchetypeVersion<span class="o">=</span>0.10-SNAPSHOT</code></pre></div>

	This allows you to <strong>name your newly created project</strong>. It will interactively ask you for the groupId, artifactId, and package name.
	</div>
	</div>

	<h2 id="inspect-project">Inspect Project</h2>

	<p>There will be a new directory in your working directory. If you’ve used the <em>curl</em> approach, the directory is called <code>quickstart</code>. Otherwise, it has the name of your artifactId.</p>

	<p>The sample project is a <strong>Maven project</strong>, which contains two classes. <em>Job</em> is a basic skeleton program and <em>WordCountJob</em> a working example. Please note that the <em>main</em> method of both classes allow you to start Flink in a development/testing mode.</p>

	<p>We recommend to <strong>import this project into your IDE</strong>. For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites:</p>

	<ul>
	<li><em>Eclipse 4.x</em>
	<ul>
	<li><a href="http://download.scala-ide.org/sdk/e38/scala210/stable/site">Scala IDE</a></li>
	<li><a href="http://alchim31.free.fr/m2e-scala/update-site">m2eclipse-scala</a></li>
	<li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.15.0/N/0.15.0.201206251206/">Build Helper Maven Plugin</a></li>
	</ul>
	</li>
	<li><em>Eclipse 3.7</em>
	<ul>
	<li><a href="http://download.scala-ide.org/sdk/e37/scala210/stable/site">Scala IDE</a></li>
	<li><a href="http://alchim31.free.fr/m2e-scala/update-site">m2eclipse-scala</a></li>
	<li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/">Build Helper Maven Plugin</a></li>
	</ul>
	</li>
	</ul>

	<p>The IntelliJ IDE also supports Maven and offers a plugin for Scala development.</p>

	<h2 id="build-project">Build Project</h2>

	<p>If you want to <strong>build your project</strong>, go to your project directory and issue the <code>mvn clean package -Pbuild-jar</code> command. You will <strong>find a jar</strong> that runs on every Flink cluster in <strong>target/your-artifact-id-1.0-SNAPSHOT.jar</strong>. There is also a fat-jar, <strong>target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar</strong>. This
	also contains all dependencies that get added to the maven project.</p>

	<h2 id="next-steps">Next Steps</h2>

	<p>Write your application!</p>

	<p>The quickstart project contains a WordCount implementation, the “Hello World” of Big Data processing systems. The goal of WordCount is to determine the frequencies of words in a text, e.g., how often do the terms “the” or “house” occurs in all Wikipedia texts.</p>

	<p><strong>Sample Input</strong>:</p>

	<div class="highlight"><pre><code class="language-bash">big data is big</code></pre></div>

	<p><strong>Sample Output</strong>:</p>

	<div class="highlight"><pre><code class="language-bash">big 2
	data 1
	is 1</code></pre></div>

	<p>The following code shows the WordCount implementation from the Quickstart which processes some text lines with two operators (FlatMap and Reduce), and writes the prints the resulting words and counts to std-out.</p>

	<div class="highlight"><pre><code class="language-scala"><span class="k">object</span> <span class="nc">WordCountJob</span> <span class="o">{</span>
	<span class="k">def</span> <span class="n">main</span><span class="o">(</span><span class="n">args</span><span class="k">:</span> <span class="kt">Array</span><span class="o">[</span><span class="kt">String</span><span class="o">])</span> <span class="o">{</span>

	<span class="c1">// set up the execution environment</span>
	<span class="k">val</span> <span class="n">env</span> <span class="k">=</span> <span class="nc">ExecutionEnvironment</span><span class="o">.</span><span class="n">getExecutionEnvironment</span>

	<span class="c1">// get input data</span>
	<span class="k">val</span> <span class="n">text</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">fromElements</span><span class="o">(</span><span class="s">"To be, or not to be,--that is the question:--"</span><span class="o">,</span>
	<span class="s">"Whether 'tis nobler in the mind to suffer"</span><span class="o">,</span> <span class="s">"The slings and arrows of outrageous fortune"</span><span class="o">,</span>
	<span class="s">"Or to take arms against a sea of troubles,"</span><span class="o">)</span>

	<span class="k">val</span> <span class="n">counts</span> <span class="k">=</span> <span class="n">text</span><span class="o">.</span><span class="n">flatMap</span> <span class="o">{</span> <span class="k">_</span><span class="o">.</span><span class="n">toLowerCase</span><span class="o">.</span><span class="n">split</span><span class="o">(</span><span class="s">"\\W+"</span><span class="o">)</span> <span class="o">}</span>
	<span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="o">(</span><span class="k">_</span><span class="o">,</span> <span class="mi">1</span><span class="o">)</span> <span class="o">}</span>
	<span class="o">.</span><span class="n">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span>
	<span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span>

	<span class="c1">// emit result</span>
	<span class="n">counts</span><span class="o">.</span><span class="n">print</span><span class="o">()</span>

	<span class="c1">// execute program</span>
	<span class="n">env</span><span class="o">.</span><span class="n">execute</span><span class="o">(</span><span class="s">"WordCount Example"</span><span class="o">)</span>
	<span class="o">}</span>
	<span class="o">}</span></code></pre></div>

	<p><a href="https://github.com/apache/flink/blob/master//flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala">Check GitHub</a> for the full example code.</p>

	<p>For a complete overview over our API, have a look at the <a href="http://flink.apache.org/docs/master/apis/programming_guide.html">Programming Guide</a> and <a href="http://flink.apache.org/docs/master/apis/examples.html">further example programs</a>. If you have any trouble, ask on our <a href="http://mail-archives.apache.org/mod_mbox/flink-dev/">Mailing List</a>. We are happy to provide help.</p>

	<h2 id="alternative-build-tools-sbt">Alternative Build Tools: SBT</h2>

	<p>To build and run applications with SBT instead of Maven is pretty straight forward. After creating the standard sbt <a href="http://www.scala-sbt.org/0.13/tutorial/Directories.html">directory layout</a> it’s enough to add the Flink dependencies to the <code>build.sbt</code> file:</p>

	<div class="highlight"><pre><code class="language-scala"><span class="n">libraryDependencies</span> <span class="o">++=</span> <span class="nc">Seq</span><span class="o">(</span><span class="s">"org.apache.flink"</span> <span class="o">%</span> <span class="s">"flink-scala"</span> <span class="o">%</span> <span class="s">"0.10-SNAPSHOT"</span><span class="o">,</span> <span class="s">"org.apache.flink"</span> <span class="o">%</span> <span class="s">"flink-clients"</span> <span class="o">%</span> <span class="s">"0.10-SNAPSHOT"</span><span class="o">)</span></code></pre></div>

	<p>Now the application can be executed by <code>sbt run</code>. By default SBT runs an application in the same JVM itself is running in. This can lead to lass loading issues with Flink. To avoid these, append the following line to <code>build.sbt</code>:</p>

	<div class="highlight"><pre><code class="language-scala"><span class="n">fork</span> <span class="n">in</span> <span class="n">run</span> <span class="o">:=</span> <span class="kc">true</span></code></pre></div>


	</div>

	<div class="col-sm-10 col-sm-offset-1">
	<!-- Disqus thread and some vertical offset -->
	<div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
	</div>
	</div>

	</div><!-- /.container -->

	<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
	<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
	<!-- Include all compiled plugins (below), or include individual files as needed -->
	<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
	<script src="http://flink.apache.org/docs/master/page/js/codetabs.js"></script>

	<!-- Google Analytics -->
	<script>
	(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]\|\|function(){
	(i[r].q=i[r].q\|\|[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','//www.google-analytics.com/analytics.js','ga');

	ga('create', 'UA-52545728-1', 'auto');
	ga('send', 'pageview');
	</script>

	<!-- Disqus -->
	<script type="text/javascript">
	var disqus_shortname = 'stratosphere-eu';
	(function() {
	var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
	dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
	(document.getElementsByTagName('head')[0] \|\| document.getElementsByTagName('body')[0]).appendChild(dsq);
	})();
	</script>
	</body>
	</html>