content/docs/flink/current/flink-streaming-netty/index.html - bahir-website - Git at Google



 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="utf-8">
     <title>Apache Flink Streaming Connector for Netty</title>
     <meta name="description" content="Apache Flink Streaming Connector for Netty">
     <meta name="author" content="">

     <!-- Enable responsive viewport -->
     <meta name="viewport" content="width=device-width, initial-scale=1.0">

     <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
     <!--[if lt IE 9]>
       <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
     <![endif]-->

     <!-- Le styles -->
     <link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet">
     <link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css">
     <link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet"  type="text/css" media="screen" />
     <!-- Le fav and touch icons -->
     <!-- Update these with your own images
     <link rel="shortcut icon" href="images/favicon.ico">
     <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
     <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
     <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
   -->

     <!-- make tables sortable by adding class tag "sortable" to table elements -->
     <script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>


   </head>

   <body>


 <!-- Navigation -->
 <div id="nav-bar">
   <nav id="nav-container" class="navbar navbar-inverse " role="navigation">
     <div class="container">
       <!-- Brand and toggle get grouped for better mobile display -->

       <div class="navbar-header page-scroll">
         <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
           <span class="sr-only">Toggle navigation</span>
           <span class="icon-bar"></span>
           <span class="icon-bar"></span>
           <span class="icon-bar"></span>
         </button>
         <a class="navbar-brand page-scroll" href="/#home">Home</a>
       </div>
       <!-- Collect the nav links, forms, and other content for toggling -->
       <nav class="navbar-collapse collapse" role="navigation">
         <ul class="nav navbar-nav">


           <li id="download">

             <a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a>
             <ul class="dropdown-menu dropdown-left">


               <li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li>


               <li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li>

             </ul>

           </li>


           <li id="community">

             <a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a>
             <ul class="dropdown-menu dropdown-left">


               <li><a href="/community" target="_self">Get Involved</a></li>


               <li><a href="/contributing" target="_self">Contributing</a></li>


               <li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li>


               <li><a href="https://issues.apache.org/jira/browse/BAHIR" target="_blank">Issue Tracker</a></li>


               <li><a href="/community#source-code" target="_self">Source Code</a></li>


               <li><a href="/community-members" target="_self">Project Committers</a></li>

             </ul>

           </li>


           <li id="documentation">

             <a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a>
             <ul class="dropdown-menu dropdown-left">


               <li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li>


               <li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li>

             </ul>

           </li>


           <li id="github">

             <a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a>
             <ul class="dropdown-menu dropdown-left">


               <li><a href="https://github.com/apache/bahir" target="_blank">Bahir Spark Extensions</a></li>


               <li><a href="https://github.com/apache/bahir-flink" target="_blank">Bahir Flink Extensions</a></li>


               <li><a href="https://github.com/apache/bahir-website" target="_blank">Bahir Website</a></li>

             </ul>

           </li>


           <li id="apache">

             <a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a>
             <ul class="dropdown-menu dropdown-left">


               <li><a href="http://www.apache.org/foundation/how-it-works.html" target="_blank">Apache Software Foundation</a></li>


               <li><a href="http://www.apache.org/licenses/" target="_blank">Apache License</a></li>


               <li><a href="http://www.apache.org/foundation/sponsorship" target="_blank">Sponsorship</a></li>


               <li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>


               <li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>

             </ul>

           </li>


         </ul>
       </nav><!--/.navbar-collapse -->
       <!-- /.navbar-collapse -->
     </div>
     <!-- /.container -->
   </nav>
 </div>


     <div class="container">


 <!--<div class="hero-unit Apache Flink Streaming Connector for Netty">
   <h1></h1>
 </div>
 -->

 <div class="row">
   <div class="col-md-12">
     <!--

 -->

 <h1 id="flink-netty-connector">Flink Netty Connector</h1>

 <p>This connector provides tcp source and http source for receiving push data, implemented by <a href="http://netty.io">Netty</a>.</p>

 <p>Note that the streaming connectors are not part of the binary distribution of Flink. You need to link them into your job jar for cluster execution.
 See how to link with them for cluster execution <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/linking.html">here</a>.</p>

 <h2 id="data-flow">Data Flow</h2>

 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>+-------------+      (2)    +------------------------+
 | user system |    &lt;-----   | Third Register Service |
 +-------------+             +------------------------+
        |                                ^
        | (3)                            |
        |                                |
        V                                |
 +--------------------+                  |
 | Flink Netty Source |  ----------------+
 +--------------------+         (1)
 </code></pre></div></div>

 <p>There are three components:</p>

 <ul>
   <li>User System - where the data stream is coming from</li>
   <li>Third Register Service - receive <code class="language-plaintext highlighter-rouge">Flink Netty Source</code>’s register request (ip and port)</li>
   <li>Flink Netty Source - Netty Server for receiving pushed streaming data from <code class="language-plaintext highlighter-rouge">User System</code></li>
 </ul>

 <h2 id="maven-dependency">Maven Dependency</h2>
 <p>To use this connector, add the following dependency to your project:</p>

 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&lt;dependency&gt;
   &lt;groupId&gt;org.apache.bahir&lt;/groupId&gt;
   &lt;artifactId&gt;flink-connector-netty_2.11&lt;/artifactId&gt;
   &lt;version&gt;1.1-SNAPSHOT&lt;/version&gt;
 &lt;/dependency&gt;
 </code></pre></div></div>

 <h2 id="usage">Usage</h2>

 <p><em>Tcp Source:</em></p>

 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>val env = StreamExecutionEnvironment.getExecutionEnvironment
 env.addSource(new TcpReceiverSource("msg", 7070, Some("http://localhost:9090/cb")))
 </code></pre></div></div>
 <blockquote>
   <p>paramKey:  the http query param key
 tryPort:   try to use this point, if this point is used then try a new port
 callbackUrl:   register connector’s ip and port to a <code class="language-plaintext highlighter-rouge">Third Register Service</code></p>
 </blockquote>

 <p><em>Http Source:</em></p>

 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>val env = StreamExecutionEnvironment.getExecutionEnvironment
 env.addSource(new TcpReceiverSource(7070, Some("http://localhost:9090/cb")))
 </code></pre></div></div>
 <blockquote>
   <p>tryPort:   try to use this port, if this point is used then try a new port
 callbackUrl:   register connector’s ip and port to a <code class="language-plaintext highlighter-rouge">Third Register Service</code></p>
 </blockquote>

 <h2 id="full-example">Full Example</h2>

 <p>There are two example to get started:</p>

 <ul>
   <li><a href="https://github.com/apache/bahir-flink/blob/master/flink-connector-netty/src/test/scala/org/apache/flink/streaming/connectors/netty/example/StreamSqlExample.scala">StreamSqlExample</a></li>
   <li><a href="https://github.com/apache/bahir-flink/blob/master/flink-connector-netty/src/test/scala/org/apache/flink/streaming/connectors/netty/example/TcpSourceExample.scala">TcpSourceExample</a></li>
 </ul>

   </div>
 </div>


       <hr>

       <!-- <p>&copy; 2021 </p>-->
       <footer class="site-footer">
     <div class="wrapper">
         <div class="footer-col-wrapper">

             <div style="text-align:center;">

                 <div>
                     Copyright &copy; 2016-<script>document.write(new Date().getFullYear());</script> <a href="http://www.apache.org">The Apache Software Foundation</a>.
                     Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
                     <br>

                     Apache and the Apache Feather logo are trademarks of The Apache Software Foundation.

                 </div>
             </div>
         </div>
     </div>
 </footer>

     </div>


   <script type="text/javascript">
   (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
   (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
   m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
   })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

   ga('create', 'UA-79140859-1', 'bahir.apache.org');
   ga('require', 'linkid', 'linkid.js');
   ga('send', 'pageview');

 </script>


     <script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script>

     <script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script>


   </body>
 </html>


	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="utf-8">
	<title>Apache Flink Streaming Connector for Netty</title>
	<meta name="description" content="Apache Flink Streaming Connector for Netty">
	<meta name="author" content="">

	<!-- Enable responsive viewport -->
	<meta name="viewport" content="width=device-width, initial-scale=1.0">

	<!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
	<!--[if lt IE 9]>
	<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
	<![endif]-->

	<!-- Le styles -->
	<link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet">
	<link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css">
	<link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" />
	<!-- Le fav and touch icons -->
	<!-- Update these with your own images
	<link rel="shortcut icon" href="images/favicon.ico">
	<link rel="apple-touch-icon" href="images/apple-touch-icon.png">
	<link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
	<link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
	-->

	<!-- make tables sortable by adding class tag "sortable" to table elements -->
	<script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>


	</head>

	<body>



	<!-- Navigation -->
	<div id="nav-bar">
	<nav id="nav-container" class="navbar navbar-inverse " role="navigation">
	<div class="container">
	<!-- Brand and toggle get grouped for better mobile display -->

	<div class="navbar-header page-scroll">
	<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
	<span class="sr-only">Toggle navigation</span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	<span class="icon-bar"></span>
	</button>
	<a class="navbar-brand page-scroll" href="/#home">Home</a>
	</div>
	<!-- Collect the nav links, forms, and other content for toggling -->
	<nav class="navbar-collapse collapse" role="navigation">
	<ul class="nav navbar-nav">



	<li id="download">

	<a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a>
	<ul class="dropdown-menu dropdown-left">


	<li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li>


	<li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li>

	</ul>

	</li>




	<li id="community">

	<a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a>
	<ul class="dropdown-menu dropdown-left">


	<li><a href="/community" target="_self">Get Involved</a></li>


	<li><a href="/contributing" target="_self">Contributing</a></li>


	<li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li>


	<li><a href="https://issues.apache.org/jira/browse/BAHIR" target="_blank">Issue Tracker</a></li>


	<li><a href="/community#source-code" target="_self">Source Code</a></li>


	<li><a href="/community-members" target="_self">Project Committers</a></li>

	</ul>

	</li>




	<li id="documentation">

	<a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a>
	<ul class="dropdown-menu dropdown-left">


	<li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li>


	<li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li>

	</ul>

	</li>




	<li id="github">

	<a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a>
	<ul class="dropdown-menu dropdown-left">


	<li><a href="https://github.com/apache/bahir" target="_blank">Bahir Spark Extensions</a></li>


	<li><a href="https://github.com/apache/bahir-flink" target="_blank">Bahir Flink Extensions</a></li>


	<li><a href="https://github.com/apache/bahir-website" target="_blank">Bahir Website</a></li>

	</ul>

	</li>




	<li id="apache">

	<a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a>
	<ul class="dropdown-menu dropdown-left">


	<li><a href="http://www.apache.org/foundation/how-it-works.html" target="_blank">Apache Software Foundation</a></li>


	<li><a href="http://www.apache.org/licenses/" target="_blank">Apache License</a></li>


	<li><a href="http://www.apache.org/foundation/sponsorship" target="_blank">Sponsorship</a></li>


	<li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>


	<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>

	</ul>

	</li>


	</ul>
	</nav><!--/.navbar-collapse -->
	<!-- /.navbar-collapse -->
	</div>
	<!-- /.container -->
	</nav>
	</div>


	<div class="container">



	<!--<div class="hero-unit Apache Flink Streaming Connector for Netty">
	<h1></h1>
	</div>
	-->

	<div class="row">
	<div class="col-md-12">
	<!--

	-->

	<h1 id="flink-netty-connector">Flink Netty Connector</h1>

	<p>This connector provides tcp source and http source for receiving push data, implemented by <a href="http://netty.io">Netty</a>.</p>

	<p>Note that the streaming connectors are not part of the binary distribution of Flink. You need to link them into your job jar for cluster execution.
	See how to link with them for cluster execution <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/linking.html">here</a>.</p>

	<h2 id="data-flow">Data Flow</h2>

	<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>+-------------+ (2) +------------------------+
	\| user system \| <----- \| Third Register Service \|
	+-------------+ +------------------------+
	\| ^
	\| (3) \|
	\| \|
	V \|
	+--------------------+ \|
	\| Flink Netty Source \| ----------------+
	+--------------------+ (1)
	</code></pre></div></div>

	<p>There are three components:</p>

	<ul>
	<li>User System - where the data stream is coming from</li>
	<li>Third Register Service - receive <code class="language-plaintext highlighter-rouge">Flink Netty Source</code>’s register request (ip and port)</li>
	<li>Flink Netty Source - Netty Server for receiving pushed streaming data from <code class="language-plaintext highlighter-rouge">User System</code></li>
	</ul>

	<h2 id="maven-dependency">Maven Dependency</h2>
	<p>To use this connector, add the following dependency to your project:</p>

	<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code><dependency>
	<groupId>org.apache.bahir</groupId>
	<artifactId>flink-connector-netty_2.11</artifactId>
	<version>1.1-SNAPSHOT</version>
	</dependency>
	</code></pre></div></div>

	<h2 id="usage">Usage</h2>

	<p><em>Tcp Source:</em></p>

	<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>val env = StreamExecutionEnvironment.getExecutionEnvironment
	env.addSource(new TcpReceiverSource("msg", 7070, Some("http://localhost:9090/cb")))
	</code></pre></div></div>
	<blockquote>
	<p>paramKey: the http query param key
	tryPort: try to use this point, if this point is used then try a new port
	callbackUrl: register connector’s ip and port to a <code class="language-plaintext highlighter-rouge">Third Register Service</code></p>
	</blockquote>

	<p><em>Http Source:</em></p>

	<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>val env = StreamExecutionEnvironment.getExecutionEnvironment
	env.addSource(new TcpReceiverSource(7070, Some("http://localhost:9090/cb")))
	</code></pre></div></div>
	<blockquote>
	<p>tryPort: try to use this port, if this point is used then try a new port
	callbackUrl: register connector’s ip and port to a <code class="language-plaintext highlighter-rouge">Third Register Service</code></p>
	</blockquote>

	<h2 id="full-example">Full Example</h2>

	<p>There are two example to get started:</p>

	<ul>
	<li><a href="https://github.com/apache/bahir-flink/blob/master/flink-connector-netty/src/test/scala/org/apache/flink/streaming/connectors/netty/example/StreamSqlExample.scala">StreamSqlExample</a></li>
	<li><a href="https://github.com/apache/bahir-flink/blob/master/flink-connector-netty/src/test/scala/org/apache/flink/streaming/connectors/netty/example/TcpSourceExample.scala">TcpSourceExample</a></li>
	</ul>

	</div>
	</div>



	<hr>

	<!-- <p>© 2021 </p>-->
	<footer class="site-footer">
	<div class="wrapper">
	<div class="footer-col-wrapper">

	<div style="text-align:center;">

	<div>
	Copyright © 2016-<script>document.write(new Date().getFullYear());</script> <a href="http://www.apache.org">The Apache Software Foundation</a>.
	Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
	<br>

	Apache and the Apache Feather logo are trademarks of The Apache Software Foundation.

	</div>
	</div>
	</div>
	</div>
	</footer>

	</div>




	<script type="text/javascript">
	(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]\|\|function(){
	(i[r].q=i[r].q\|\|[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','//www.google-analytics.com/analytics.js','ga');

	ga('create', 'UA-79140859-1', 'bahir.apache.org');
	ga('require', 'linkid', 'linkid.js');
	ga('send', 'pageview');

	</script>



	<script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script>

	<script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script>


	</body>
	</html>