blob: b1050d231b5347739124955dcf1de05039dd7c67 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2023/08/04/announcing-three-new-apache-flink-connectors-the-new-connector-versioning-strategy-and-externalization/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="New connectors # We&rsquo;re excited to announce that Apache Flink now supports three new connectors: Amazon DynamoDB, MongoDB and OpenSearch! The connectors are available for both the DataStream and Table/SQL APIs.
Amazon DynamoDB - This connector includes a sink that provides at-least-once delivery guarantees. MongoDB connector - This connector includes a source and sink that provide at-least-once guarantees. OpenSearch sink - This connector includes a sink that provides at-least-once guarantees.">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Announcing three new Apache Flink connectors, the new connector versioning strategy and externalization" />
<meta property="og:description" content="New connectors # We&rsquo;re excited to announce that Apache Flink now supports three new connectors: Amazon DynamoDB, MongoDB and OpenSearch! The connectors are available for both the DataStream and Table/SQL APIs.
Amazon DynamoDB - This connector includes a sink that provides at-least-once delivery guarantees. MongoDB connector - This connector includes a source and sink that provide at-least-once guarantees. OpenSearch sink - This connector includes a sink that provides at-least-once guarantees." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2023/08/04/announcing-three-new-apache-flink-connectors-the-new-connector-versioning-strategy-and-externalization/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2023-08-04T12:00:00+00:00" />
<meta property="article:modified_time" content="2023-08-04T12:00:00+00:00" />
<title>Announcing three new Apache Flink connectors, the new connector versioning strategy and externalization | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2023/08/04/announcing-three-new-apache-flink-connectors-the-new-connector-versioning-strategy-and-externalization/">Announcing three new Apache Flink connectors, the new connector versioning strategy and externalization</a>
</h1>
August 4, 2023 -
Elphas Toringepi
<a href="https://twitter.com/elphastori">(@elphastori)</a>
<p><h2 id="new-connectors">
New connectors
<a class="anchor" href="#new-connectors">#</a>
</h2>
<p>We&rsquo;re excited to announce that Apache Flink now supports three new connectors: <a href="https://aws.amazon.com/dynamodb">Amazon DynamoDB</a>, <a href="https://www.mongodb.com/">MongoDB</a> and <a href="https://opensearch.org/">OpenSearch</a>! The connectors are available for both the <a href="https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/overview/">DataStream</a> and <a href="https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/table/overview/">Table/SQL</a> APIs.</p>
<ul>
<li><strong><a href="https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/dynamodb/">Amazon DynamoDB</a></strong> - This connector includes a sink that provides at-least-once delivery guarantees.</li>
<li><strong><a href="https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/mongodb/">MongoDB connector</a></strong> - This connector includes a source and sink that provide at-least-once guarantees.</li>
<li><strong><a href="https://github.com/apache/flink-connector-opensearch/blob/main/docs/content/docs/connectors/datastream/opensearch.md">OpenSearch sink</a></strong> - This connector includes a sink that provides at-least-once guarantees.</li>
</ul>
<table>
<thead>
<tr>
<th>Connector</th>
<th>Date Released</th>
<th>Supported Flink Versions</th>
</tr>
</thead>
<tbody>
<tr>
<td>Amazon DynamoDB sink</td>
<td>2022-12-02</td>
<td>1.15+</td>
</tr>
<tr>
<td>MongoDB connector</td>
<td>2023-03-31</td>
<td>1.16+</td>
</tr>
<tr>
<td>OpenSearch sink</td>
<td>2022-12-21</td>
<td>1.16+</td>
</tr>
</tbody>
</table>
<h3 id="list-of-contributors">
List of Contributors
<a class="anchor" href="#list-of-contributors">#</a>
</h3>
<p>The Apache Flink community would like to express gratitude to all the new connector contributors:</p>
<p>Andriy Redko, Chesnay Schepler, Danny Cranmer, darenwkt, Hong Liang Teoh, Jiabao Sun, Leonid Ilyevsky, Martijn Visser, nir.tsruya, Sergey Nuyanzin, Weijie Guo, Yuri Gusev, Yuxin Tan</p>
<h2 id="externalized-connectors">
Externalized connectors
<a class="anchor" href="#externalized-connectors">#</a>
</h2>
<p>The community has externalized connectors from <a href="https://github.com/apache/flink">Flink’s main repository</a>. This was driven to realise the following benefits:</p>
<ul>
<li><strong>Faster releases of connectors:</strong> New features can be added more quickly, bugs can be fixed immediately, and we can have faster security patches in case of direct or indirect (through dependencies) security flaws.</li>
<li><strong>Adding newer connector features to older Flink versions:</strong> By having stable connector APIs, the same connector artifact may be used with different Flink versions. Thus, new features can also immediately be used with older Flink versions.</li>
<li><strong>More activity and contributions around connectors:</strong> By easing the contribution and development process around connectors, we will see faster development and also more connectors.</li>
<li><strong>Documentation:</strong> Standardized documentation and user experience for the connectors, regardless of where they are maintained.</li>
<li><strong>A faster Flink CI:</strong> By not needing to build and test connectors, the Flink CI pipeline will be faster and Flink developers will experience fewer build instabilities (which mostly come from connectors). That should speed up Flink development.</li>
</ul>
<p>The following connectors have been moved to individual repositories:</p>
<ul>
<li><a href="https://github.com/apache/flink-connector-kafka">Kafka / Upsert-Kafka</a></li>
<li><a href="https://github.com/apache/flink-connector-cassandra">Cassandra</a></li>
<li><a href="https://github.com/apache/flink-connector-elasticsearch/">Elasticsearch</a></li>
<li><a href="https://github.com/apache/flink-connector-mongodb"><strong>MongoDB</strong></a></li>
<li><strong><a href="https://github.com/apache/flink-connector-opensearch">OpenSearch</a></strong></li>
<li><a href="https://github.com/apache/flink-connector-rabbitmq">RabbitMQ</a></li>
<li><a href="https://github.com/apache/flink-connector-gcp-pubsub">Google Cloud PubSub</a></li>
<li><a href="https://github.com/apache/flink-connector-pulsar/">Pulsar</a></li>
<li><a href="https://github.com/apache/flink-connector-jdbc">JDBC</a></li>
<li><a href="https://github.com/apache/flink-connector-hbase">HBase</a></li>
<li><a href="https://github.com/apache/flink-connector-hive">Hive</a></li>
<li><a href="https://github.com/apache/flink-connector-aws">AWS connectors</a>:
<ul>
<li>Firehose</li>
<li>Kinesis</li>
<li><strong>DynamoDB</strong></li>
</ul>
</li>
</ul>
<h3 id="versioning">
Versioning
<a class="anchor" href="#versioning">#</a>
</h3>
<p>Connectors continue to use the same Maven dependency <code>groupId</code> and <code>artifactId</code>. However, the JAR artifact <code>version</code> has changed and now uses the format, <code>&lt;major&gt;.&lt;minor&gt;.&lt;patch&gt;-&lt;flink-major&gt;.&lt;flink-minor&gt;</code>. For example, to use the DynamoDB connector for Flink 1.17, add the following dependency to your project:</p>
<pre tabindex="0"><code>&lt;dependency&gt;
&lt;groupId&gt;org.apache.flink&lt;/groupId&gt;
&lt;artifactId&gt;flink-connector-dynamodb&lt;/artifactId&gt;
&lt;version&gt;4.1.0-1.17&lt;/version&gt;
&lt;/dependency&gt;
</code></pre><p>You can find the maven dependency for a connector in the <a href="https://nightlies.apache.org/flink/flink-docs-release-1.17/docs/connectors/datastream/overview">Flink connectors documentation</a> for a specific Flink version. Use the <a href="https://flink.apache.org/downloads">Flink Downloads page</a> to verify which version your connector is compatible with.</p>
<h3 id="contributing">
Contributing
<a class="anchor" href="#contributing">#</a>
</h3>
<p>Similarly, when creating JIRAs to report issues or to contribute to externalized connectors, the <code>Affects Version/s</code> and <code>Fix Version/s</code> fields should now use the connector version instead of a Flink version. The format should be <code>&lt;connector-name&gt;-&lt;major&gt;.&lt;minor&gt;.&lt;patch&gt;</code>. For example, use <code>opensearch-1.1.0</code> for the OpenSearch connector. All other fields in the JIRA like <code>Component/s</code> remain the same.</p>
<p>For more information on how to contribute to externalized connectors, see the <a href="https://cwiki.apache.org/confluence/display/FLINK/Externalized&#43;Connector&#43;development">Externalized Connector development wiki</a>.</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2023-08-04-externalized-connectors.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#new-connectors">New connectors</a>
<ul>
<li><a href="#list-of-contributors">List of Contributors</a></li>
</ul>
</li>
<li><a href="#externalized-connectors">Externalized connectors</a>
<ul>
<li><a href="#versioning">Versioning</a></li>
<li><a href="#contributing">Contributing</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>