blob: c3f83d84765357c87d5084a4626044b6290103c3 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Apache Flink Streaming Connector for Redis</title>
<meta name="description" content="Apache Flink Streaming Connector for Redis">
<meta name="author" content="">
<!-- Enable responsive viewport -->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
<!--[if lt IE 9]>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<!-- Le styles -->
<link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet">
<link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css">
<link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" />
<!-- Le fav and touch icons -->
<!-- Update these with your own images
<link rel="shortcut icon" href="images/favicon.ico">
<link rel="apple-touch-icon" href="images/apple-touch-icon.png">
<link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
<link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
-->
<!-- make tables sortable by adding class tag "sortable" to table elements -->
<script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>
</head>
<body>
<!-- Navigation -->
<div id="nav-bar">
<nav id="nav-container" class="navbar navbar-inverse " role="navigation">
<div class="container">
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header page-scroll">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand page-scroll" href="/#home">Home</a>
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<nav class="navbar-collapse collapse" role="navigation">
<ul class="nav navbar-nav">
<li id="download">
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a>
<ul class="dropdown-menu dropdown-left">
<li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li>
<li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li>
</ul>
</li>
<li id="community">
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a>
<ul class="dropdown-menu dropdown-left">
<li><a href="/community" target="_self">Get Involved</a></li>
<li><a href="/contributing" target="_self">Contributing</a></li>
<li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li>
<li><a href="https://issues.apache.org/jira/browse/BAHIR" target="_blank">Issue Tracker</a></li>
<li><a href="/community#source-code" target="_self">Source Code</a></li>
<li><a href="/community-members" target="_self">Project Committers</a></li>
</ul>
</li>
<li id="documentation">
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a>
<ul class="dropdown-menu dropdown-left">
<li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li>
<li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li>
</ul>
</li>
<li id="github">
<a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a>
<ul class="dropdown-menu dropdown-left">
<li><a href="https://github.com/apache/bahir" target="_blank">Bahir Spark Extensions</a></li>
<li><a href="https://github.com/apache/bahir-flink" target="_blank">Bahir Flink Extensions</a></li>
<li><a href="https://github.com/apache/bahir-website" target="_blank">Bahir Website</a></li>
</ul>
</li>
<li id="apache">
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a>
<ul class="dropdown-menu dropdown-left">
<li><a href="http://www.apache.org/foundation/how-it-works.html" target="_blank">Apache Software Foundation</a></li>
<li><a href="http://www.apache.org/licenses/" target="_blank">Apache License</a></li>
<li><a href="http://www.apache.org/foundation/sponsorship" target="_blank">Sponsorship</a></li>
<li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>
</ul>
</li>
</ul>
</nav><!--/.navbar-collapse -->
<!-- /.navbar-collapse -->
</div>
<!-- /.container -->
</nav>
</div>
<div class="container">
<!--<div class="hero-unit Apache Flink Streaming Connector for Redis">
<h1></h1>
</div>
-->
<div class="row">
<div class="col-md-12">
<!--
-->
<h1 id="flink-redis-connector">Flink Redis Connector</h1>
<p>This connector provides a Sink that can write to <a href="http://redis.io/">Redis</a> and also can publish data
to <a href="http://redis.io/topics/pubsub">Redis PubSub</a>. To use this connector, add the
following dependency to your project:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&lt;dependency&gt;
&lt;groupId&gt;org.apache.bahir&lt;/groupId&gt;
&lt;artifactId&gt;flink-connector-redis_2.11&lt;/artifactId&gt;
&lt;version&gt;1.0&lt;/version&gt;
&lt;/dependency&gt;
</code></pre></div></div>
<p><em>Version Compatibility</em>: This module is compatible with Redis 2.8.5.</p>
<p>Note that the streaming connectors are not part of the binary distribution of Flink. You need to link them into your job jar for cluster execution.
See how to link with them for cluster execution <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/linking.html">here</a>.</p>
<h2 id="installing-redis">Installing Redis</h2>
<p>Follow the instructions from the <a href="http://redis.io/download">Redis download page</a>.</p>
<h2 id="redis-sink">Redis Sink</h2>
<p>A class providing an interface for sending data to Redis.
The sink can use three different methods for communicating with different type of Redis environments:</p>
<ol>
<li>Single Redis Server</li>
<li>Redis Cluster</li>
<li>Redis Sentinel</li>
</ol>
<p>This code shows how to create a sink that communicate to a single redis server:</p>
<p><strong>Java:</strong></p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>public static class RedisExampleMapper implements RedisMapper&lt;Tuple2&lt;String, String&gt;&gt;{
@Override
public RedisCommandDescription getCommandDescription() {
return new RedisCommandDescription(RedisCommand.HSET, "HASH_NAME");
}
@Override
public String getKeyFromData(Tuple2&lt;String, String&gt; data) {
return data.f0;
}
@Override
public String getValueFromData(Tuple2&lt;String, String&gt; data) {
return data.f1;
}
}
FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder().setHost("127.0.0.1").build();
DataStream&lt;String&gt; stream = ...;
stream.addSink(new RedisSink&lt;Tuple2&lt;String, String&gt;&gt;(conf, new RedisExampleMapper());
</code></pre></div></div>
<p><strong>Scala:</strong></p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>class RedisExampleMapper extends RedisMapper[(String, String)]{
override def getCommandDescription: RedisCommandDescription = {
new RedisCommandDescription(RedisCommand.HSET, "HASH_NAME")
}
override def getKeyFromData(data: (String, String)): String = data._1
override def getValueFromData(data: (String, String)): String = data._2
}
val conf = new FlinkJedisPoolConfig.Builder().setHost("127.0.0.1").build()
stream.addSink(new RedisSink[(String, String)](conf, new RedisExampleMapper))
</code></pre></div></div>
<p>This example code does the same, but for Redis Cluster:</p>
<p><strong>Java:</strong></p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder()
.setNodes(new HashSet&lt;InetSocketAddress&gt;(Arrays.asList(new InetSocketAddress(5601)))).build();
DataStream&lt;String&gt; stream = ...;
stream.addSink(new RedisSink&lt;Tuple2&lt;String, String&gt;&gt;(conf, new RedisExampleMapper());
</code></pre></div></div>
<p><strong>Scala:</strong></p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>val conf = new FlinkJedisPoolConfig.Builder().setNodes(...).build()
stream.addSink(new RedisSink[(String, String)](conf, new RedisExampleMapper))
</code></pre></div></div>
<p>This example shows when the Redis environment is with Sentinels:</p>
<p>Java:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>FlinkJedisSentinelConfig conf = new FlinkJedisSentinelConfig.Builder()
.setMasterName("master").setSentinels(...).build();
DataStream&lt;String&gt; stream = ...;
stream.addSink(new RedisSink&lt;Tuple2&lt;String, String&gt;&gt;(conf, new RedisExampleMapper());
</code></pre></div></div>
<p>Scala:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>val conf = new FlinkJedisSentinelConfig.Builder().setMasterName("master").setSentinels(...).build()
stream.addSink(new RedisSink[(String, String)](conf, new RedisExampleMapper))
</code></pre></div></div>
<p>This section gives a description of all the available data types and what Redis command used for that.</p>
<table class="table table-bordered" style="width: 75%">
<thead>
<tr>
<th class="text-center" style="width: 20%">Data Type</th>
<th class="text-center" style="width: 25%">Redis Command [Sink]</th>
</tr>
</thead>
<tbody>
<tr>
<td>HASH</td><td><a href="http://redis.io/commands/hset">HSET</a></td>
</tr>
<tr>
<td>LIST</td><td>
<a href="http://redis.io/commands/rpush">RPUSH</a>,
<a href="http://redis.io/commands/lpush">LPUSH</a>
</td>
</tr>
<tr>
<td>SET</td><td><a href="http://redis.io/commands/rpush">SADD</a></td>
</tr>
<tr>
<td>PUBSUB</td><td><a href="http://redis.io/commands/publish">PUBLISH</a></td>
</tr>
<tr>
<td>STRING</td><td><a href="http://redis.io/commands/set">SET</a></td>
</tr>
<tr>
<td>HYPER_LOG_LOG</td><td><a href="http://redis.io/commands/pfadd">PFADD</a></td>
</tr>
<tr>
<td>SORTED_SET</td><td><a href="http://redis.io/commands/zadd">ZADD</a></td>
</tr>
<tr>
<td>SORTED_SET</td><td><a href="http://redis.io/commands/zrem">ZREM</a></td>
</tr>
</tbody>
</table>
</div>
</div>
<hr>
<!-- <p>&copy; 2021 </p>-->
<footer class="site-footer">
<div class="wrapper">
<div class="footer-col-wrapper">
<div style="text-align:center;">
<div>
Copyright &copy; 2016-<script>document.write(new Date().getFullYear());</script> <a href="http://www.apache.org">The Apache Software Foundation</a>.
Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
<br>
Apache and the Apache Feather logo are trademarks of The Apache Software Foundation.
</div>
</div>
</div>
</div>
</footer>
</div>
<script type="text/javascript">
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-79140859-1', 'bahir.apache.org');
ga('require', 'linkid', 'linkid.js');
ga('send', 'pageview');
</script>
<script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script>
<script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script>
</body>
</html>