| <!DOCTYPE html> |
| <head> |
| |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content=""> |
| <meta name="keywords" content=" "> |
| <title>Common Quarks operations | Apache Quarks Documentation</title> |
| <link rel="stylesheet" type="text/css" href="../css/syntax.css"> |
| <link rel="stylesheet" type="text/css" href="../css/font-awesome.min.css"> |
| <!--<link rel="stylesheet" type="text/css" href="../css/bootstrap.min.css">--> |
| <link rel="stylesheet" type="text/css" href="../css/modern-business.css"> |
| <link rel="stylesheet" type="text/css" href="../css/lavish-bootstrap.css"> |
| <link rel="stylesheet" type="text/css" href="../css/customstyles.css"> |
| <link rel="stylesheet" type="text/css" href="../css/theme-blue.css"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-cookie/1.4.1/jquery.cookie.min.js"></script> |
| <script src="../js/jquery.navgoco.min.js"></script> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/2.0.0/anchor.min.js"></script> |
| <script src="../js/toc.js"></script> |
| <script src="../js/customscripts.js"></script> |
| <link rel="shortcut icon" href="../common_images/favicon.ico" type="image/x-icon"> |
| <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script> |
| <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <script> |
| $(function () { |
| $('[data-toggle="tooltip"]').tooltip() |
| }) |
| </script> |
| |
| |
| |
| </head> |
| |
| <body> |
| |
| <!-- Navigation --> |
| <nav class="navbar navbar-inverse navbar-fixed-top" role="navigation"> |
| <div class="container topnavlinks"> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="sr-only">Toggle navigation</span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| |
| <a class="fa fa-home fa-lg navbar-brand" href="../docs/home.html"> <span class="projectTitle"> Apache Quarks Documentation</span></a> |
| |
| </div> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-right"> |
| <!-- entries without drop-downs appear here --> |
| <!-- conditional logic to control which topnav appears for the audience defined in the configuration file.--> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="dropdown"> |
| |
| |
| |
| <a href="#" class="dropdown-toggle" data-toggle="dropdown">GitHub Repos<b class="caret"></b></a> |
| <ul class="dropdown-menu"> |
| |
| |
| |
| <li><a href="https://github.com/apache/incubator-quarks" target="_blank">Source code</a></li> |
| |
| |
| |
| |
| |
| <li><a href="https://github.com/apache/incubator-quarks-website" target="_blank">Website/Documentation</a></li> |
| |
| |
| |
| |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="https://quarks-edge.github.io/quarks/docs/javadoc/index.html" target="_blank">Javadoc</a></li> |
| |
| |
| |
| |
| |
| |
| <!-- entries with drop-downs appear here --> |
| <!-- conditional logic to control which topnav appears for the audience defined in the configuration file.--> |
| |
| <li class="dropdown"> |
| |
| |
| |
| <a href="#" class="dropdown-toggle" data-toggle="dropdown">Quarks Resources<b class="caret"></b></a> |
| <ul class="dropdown-menu"> |
| |
| |
| |
| <li><a href="https://github.com/apache/incubator-quarks/releases" target="_blank">Download</a></li> |
| |
| |
| |
| |
| |
| <li><a href="samples">Samples</a></li> |
| |
| |
| |
| |
| |
| <li><a href="faq">FAQ</a></li> |
| |
| |
| |
| |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| <!-- special insertion --> |
| |
| |
| <!-- Send feedback function --> |
| <script> |
| function SendLinkByMail(href) { |
| var subject= "Apache Quarks Documentation feedback"; |
| var body = "I have some feedback about the Common Quarks operations page: "; |
| body += window.location.href; |
| body += ""; |
| var uri = "mailto:?subject="; |
| uri += encodeURIComponent(subject); |
| uri += "&body="; |
| uri += encodeURIComponent(body); |
| window.location.href = uri; |
| } |
| </script> |
| |
| <li><a href="mailto:dev@quarks.incubator.apache.org" target="_blank"><i class="fa fa-envelope-o"></i> Feedback</a></li> |
| |
| |
| <!--uncomment this block if you want simple search instead of algolia--> |
| <li> |
| <!--start search--> |
| <div id="search-demo-container"> |
| <input type="text" id="search-input" placeholder="search..."> |
| <ul id="results-container"></ul> |
| </div> |
| <script src="../js/jekyll-search.js" type="text/javascript"></script> |
| <script type="text/javascript"> |
| SimpleJekyllSearch.init({ |
| searchInput: document.getElementById('search-input'), |
| resultsContainer: document.getElementById('results-container'), |
| dataSource: '../search.json', |
| searchResultTemplate: '<li><a href="{url}" title="Common Quarks operations">{title}</a></li>', |
| noResultsText: 'No results found.', |
| limit: 10, |
| fuzzy: true, |
| }) |
| </script> |
| <!--end search--> |
| </li> |
| |
| |
| </div> |
| <!-- /.container --> |
| </nav> |
| |
| |
| |
| <!-- Page Content --> |
| <div class="container"> |
| <div class="col-lg-12"> </div> |
| |
| |
| <!-- Content Row --> |
| <div class="row"> |
| <!-- Sidebar Column --> |
| <div class="col-md-3"> |
| |
| <script> |
| |
| $(document).ready(function() { |
| // Initialize navgoco with default options |
| $("#mysidebar").navgoco({ |
| caretHtml: '', |
| accordion: true, |
| openClass: 'active', // open |
| save: true, |
| cookie: { |
| name: 'navgoco', |
| expires: false, |
| path: '/' |
| }, |
| slide: { |
| duration: 400, |
| easing: 'swing' |
| } |
| }); |
| |
| $("#collapseAll").click(function(e) { |
| e.preventDefault(); |
| $("#mysidebar").navgoco('toggle', false); |
| }); |
| |
| $("#expandAll").click(function(e) { |
| e.preventDefault(); |
| $("#mysidebar").navgoco('toggle', true); |
| }); |
| |
| }); |
| |
| </script> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <ul id="mysidebar" class="nav"> |
| |
| <span class="siteTagline">Quarks</span> |
| <span class="versionTagline">Version 0.3.0</span> |
| |
| |
| |
| |
| |
| |
| <li><a href="#">Overview</a> |
| <ul> |
| |
| |
| |
| |
| <li><a href="../docs/quarks_index.html">Introduction</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../docs/faq.html">FAQ</a></li> |
| |
| |
| |
| |
| |
| |
| </ul> |
| |
| |
| |
| <li><a href="#">Get Started</a> |
| <ul> |
| |
| |
| |
| |
| <li><a href="../docs/quarks-getting-started.html">Getting started guide</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="active"><a href="../docs/common-quarks-operations.html">Common operations</a></li> |
| |
| |
| |
| |
| |
| |
| </ul> |
| |
| |
| |
| <li><a href="#">Quarks Cookbook</a> |
| <ul> |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_hello_quarks.html">Hello Quarks!</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_source_function.html">Writing a source function</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_value_out_of_range.html">Detecting a sensor value out of expected range</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_different_processing_against_stream.html">Applying different processing against a single stream</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_combining_streams_processing_results.html">Splitting a stream to apply different processing and combining the results into a single stream</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_external_filter_range.html">Using an external configuration file for filter ranges</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_adaptable_filter_range.html">Changing a filter's range</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_adaptable_polling_source.html">Changing a polled source stream's period</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_adaptable_deadtime_filter.html">Using an adaptable deadtime filter</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_dynamic_analytic_control.html">Dynamically enabling analytic flows</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_parallel_analytics.html">How can I run analytics on several tuples in parallel?</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../recipes/recipe_concurrent_analytics.html">How can I run several analytics on a tuple concurrently?</a></li> |
| |
| |
| |
| |
| |
| |
| </ul> |
| |
| |
| |
| <li><a href="#">Sample Programs</a> |
| <ul> |
| |
| |
| |
| |
| <li><a href="../docs/samples.html">Samples</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../docs/quickstart.html">Quickstart IBM Watson IoT Platform</a></li> |
| |
| |
| |
| |
| |
| |
| </ul> |
| |
| |
| |
| <li><a href="#">Using the Console</a> |
| <ul> |
| |
| |
| |
| |
| <li><a href="../docs/console.html">Using the console</a></li> |
| |
| |
| |
| |
| |
| |
| </ul> |
| |
| |
| |
| <li><a href="#">Get Involved</a> |
| <ul> |
| |
| |
| |
| |
| <li><a href="../docs/community.html">How to participate</a></li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li><a href="../docs/committers.html">Committers</a></li> |
| |
| |
| |
| |
| |
| |
| </ul> |
| |
| |
| |
| |
| |
| <!-- if you aren't using the accordion, uncomment this block: |
| |
| <p class="external"> |
| <a href="#" id="collapseAll">Collapse All</a> | <a href="#" id="expandAll">Expand All</a> |
| </p> |
| --> |
| <br/> |
| </li> |
| </ul> |
| <div class="row"> |
| <div class="col-md-12"> |
| |
| <!-- this handles the automatic toc. use ## for subheads to auto-generate the on-page minitoc. if you use html tags, you must supply an ID for the heading element in order for it to appear in the minitoc. --> |
| <script> |
| $( document ).ready(function() { |
| // Handler for .ready() called. |
| |
| $('#toc').toc({ minimumHeaders: 0, listType: 'ul', showSpeed: 0, headers: 'h2,h3,h4' }); |
| |
| /* this offset helps account for the space taken up by the floating toolbar. */ |
| $('#toc').on('click', 'a', function() { |
| var target = $(this.getAttribute('href')) |
| , scroll_target = target.offset().top |
| |
| $(window).scrollTop(scroll_target - 10); |
| return false |
| }) |
| |
| }); |
| </script> |
| |
| |
| <div id="toc"></div> |
| |
| </div> |
| </div> |
| </div> |
| |
| <!-- this highlights the active parent class in the navgoco sidebar. this is critical so that the parent expands when you're viewing a page. This must appear below the sidebar code above. Otherwise, if placed inside customscripts.js, the script runs before the sidebar code runs and the class never gets inserted.--> |
| <script>$("li.active").parents('li').toggleClass("active");</script> |
| |
| |
| <!-- Content Column --> |
| <div class="col-md-9"> |
| |
| <div class="post-header"> |
| <h1 class="post-title-main">Common Quarks operations</h1> |
| </div> |
| |
| <div class="post-content"> |
| |
| |
| |
| <!-- this handles the automatic toc. use ## for subheads to auto-generate the on-page minitoc. if you use html tags, you must supply an ID for the heading element in order for it to appear in the minitoc. --> |
| <script> |
| $( document ).ready(function() { |
| // Handler for .ready() called. |
| |
| $('#toc').toc({ minimumHeaders: 0, listType: 'ul', showSpeed: 0, headers: 'h2,h3,h4' }); |
| |
| /* this offset helps account for the space taken up by the floating toolbar. */ |
| $('#toc').on('click', 'a', function() { |
| var target = $(this.getAttribute('href')) |
| , scroll_target = target.offset().top |
| |
| $(window).scrollTop(scroll_target - 10); |
| return false |
| }) |
| |
| }); |
| </script> |
| |
| |
| <div id="toc"></div> |
| |
| |
| |
| |
| <a target="_blank" href="https://github.com/apache/incubator-quarks-website/blob/master/site/docs/common-quarks-operations.md" class="btn btn-default githubEditButton" role="button"><i class="fa fa-github fa-lg"></i> Edit me</a> |
| |
| <p>In the <a href="quarks-getting-started">Getting started guide</a>, we covered a Quarks application where we read from a device's simulated temperature sensor. Yet Quarks supports more operations than simple filtering. Data analysis and streaming require a suite of functionality, the most important components of which will be outlined below.</p> |
| |
| <h2 id="tstream-map">TStream.map()</h2> |
| |
| <p><code>TStream.map()</code> is arguably the most used method in the Quarks API. Its two main purposes are to perform stateful or stateless operations on a stream's tuples, and to produce a <code>TStream</code> with tuples of a different type from that of the calling stream.</p> |
| |
| <h3 id="changing-a-tstream-39-s-tuple-type">Changing a TStream's tuple type</h3> |
| |
| <p>In addition to filtering tuples, <code>TStream</code>s support operations that <em>transform</em> tuples from one Java type to another by invoking the <code>TStream.map()</code> method.</p> |
| |
| <p><img src="images/Map_Type_Change.jpg" alt="Image of a type change" style="width:750px; height:150px;"></p> |
| |
| <p>This is useful in cases such as calculating the floating point average of a list of <code>Integer</code>s, or tokenizing a Java String into a list of <code>String</code>s. To demonstrate this, let's say we have a <code>TStream</code> which contains a few lines, each of which contains multiple words:</p> |
| <div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TStream</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">lines</span> <span class="o">=</span> <span class="n">topology</span><span class="o">.</span><span class="na">strings</span><span class="o">(</span> |
| <span class="s">"this is a line"</span><span class="o">,</span> |
| <span class="s">"this is another line"</span><span class="o">,</span> |
| <span class="s">"there are three lines now"</span><span class="o">,</span> |
| <span class="s">"and now four"</span> |
| <span class="o">);</span> |
| </code></pre></div> |
| <p>We then want to print the third word in each line. The best way to do this is to convert each line to a list of <code>String</code>s by tokenizing them. We can do this in one line of code with the <code>TStream.map()</code> method:</p> |
| <div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TStream</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="o">></span> <span class="n">wordsInLine</span> <span class="o">=</span> <span class="n">lines</span><span class="o">.</span><span class="na">map</span><span class="o">(</span><span class="n">tuple</span> <span class="o">-></span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span><span class="n">tuple</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">)));</span> |
| </code></pre></div> |
| <p>Since each tuple is now a list of strings, the <code>wordsInLine</code> stream is of type <code>List<String></code>. As you can see, the <code>map()</code> method has the ability to change the type of the <code>TStream</code>. Finally, we can use the <code>wordsInLine</code> stream to print the third word in each line.</p> |
| <div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">wordsInLine</span><span class="o">.</span><span class="na">sink</span><span class="o">(</span><span class="n">list</span> <span class="o">-></span> <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">list</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="mi">2</span><span class="o">)));</span> |
| </code></pre></div> |
| <p>As mentioned in the <a href="quarks-getting-started">Getting started guide</a>, a <code>TStream</code> can be parameterized to any serializable Java type, including ones created by the user.</p> |
| |
| <h3 id="performing-stateful-operations">Performing stateful operations</h3> |
| |
| <p>In all previous examples, the operations performed on a <code>TStream</code> have been stateless; keeping track of information over multiple invocations of the same operation has not been necessary. What if we want to keep track of the number of Strings sent over a stream? To do this, we need our <code>TStream.map()</code> method to contain a counter as state.</p> |
| |
| <p><img src="images/Map_Stateful.jpg" alt="Image of a stateful operation" style="width:750px; height:150px;"></p> |
| |
| <p>This can be achieved by creating an anonymous <code>Function</code> class, and giving it the required fields.</p> |
| <div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TStream</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">streamOfStrings</span> <span class="o">=</span> <span class="o">...;</span> |
| <span class="n">TStream</span><span class="o"><</span><span class="n">Integer</span><span class="o">></span> <span class="n">counts</span> <span class="o">=</span> <span class="n">streamOfStrings</span><span class="o">.</span><span class="na">map</span><span class="o">(</span><span class="k">new</span> <span class="n">Function</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>()</span> <span class="o">{</span> |
| <span class="kt">int</span> <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span><span class="o">;</span> |
| <span class="nd">@Override</span> |
| <span class="kd">public</span> <span class="n">Integer</span> <span class="n">apply</span><span class="o">(</span><span class="n">String</span> <span class="n">arg0</span><span class="o">)</span> <span class="o">{</span> |
| <span class="n">count</span> <span class="o">=</span> <span class="n">count</span> <span class="o">+</span> <span class="mi">1</span><span class="o">;</span> |
| <span class="k">return</span> <span class="n">count</span><span class="o">;</span> |
| <span class="o">}</span> |
| <span class="o">});</span> |
| </code></pre></div> |
| <p>The <code>count</code> field will now contain the number of <code>String</code>s which were sent over <code>streamOfStrings</code>. Although this is a simple example, the anonymous <code>Function</code> passed to <code>TStream.map()</code> can contain any kind of state! This could be a <code>HashMap<K,V></code>, a running list of tuples, or any serializable Java type. The state will be maintained throughout the entire runtime of your application.</p> |
| |
| |
| <div class="tags"> |
| |
| </div> |
| |
| <!-- |
| |
| <div id="disqus_thread"></div> |
| <script type="text/javascript"> |
| /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ |
| var disqus_shortname = 'idrbwjekyll'; // required: replace example with your forum shortname |
| |
| /* * * DON'T EDIT BELOW THIS LINE * * */ |
| (function() { |
| var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; |
| dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; |
| (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); |
| })(); |
| </script> |
| <noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript> |
| --> |
| |
| </div> |
| |
| |
| |
| <footer> |
| <div class="row"> |
| <div class="col-lg-12 footer"> |
| |
| Site last |
| generated: May 9, 2016 <br/> |
| |
| </div> |
| </div> |
| <br/> |
| <div class="row"> |
| <div class="col-md-12"> |
| <p class="small">Apache Quarks is an effort undergoing Incubation at The Apache Software |
| Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects |
| until a further review indicates that the infrastructure, communications, and decision making process |
| have stabilized in a manner consistent with other successful ASF projects. While incubation status is |
| not necessarily a reflection of the completeness or stability of the code, it does indicate that the |
| project has yet to be fully endorsed by the ASF.</p> |
| </div> |
| </div> |
| <div class="row"> |
| <div class="col-md-12"> |
| <p class="small">Copyright © 2016 The Apache Software Foundation. Licensed under the Apache |
| License, Version 2.0. |
| Apache, the Apache Feather logo, and the Apache Incubator project logo are trademarks of The Apache |
| Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their |
| respective owners.</p> |
| </div> |
| </div> |
| </footer> |
| |
| </div><!-- /.row --> |
| |
| </div> <!-- /.container --> |
| |
| </body> |
| |
| |
| </html> |
| |
