Google Git
Sign in
apache / pulsar-site / aee2d9089b6b8e327f04261353077211f4abd636 / . / content / blog / 2019 / 02 / 20 / Apache-Pulsar-2-3-0.html
blob: bcd179587cf99878ea02a6050cb6a756fc9eb7b7 [file] [log] [blame]
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Apache Pulsar 2.3.0 · Apache Pulsar</title><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="generator" content="Docusaurus"/><meta name="description" content="The Apache Pulsar PMC is happy to announce the release of Pulsar 2.3.0. This"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="Apache Pulsar 2.3.0 · Apache Pulsar"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pulsar.apache.org/blog/2019/02/20/Apache-Pulsar-2-3-0"/><meta property="og:description" content="The Apache Pulsar PMC is happy to announce the release of Pulsar 2.3.0. This"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pulsar.apache.org/img/pulsar.svg"/><link rel="shortcut icon" href="/img/pulsar.ico"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/atom-one-dark.min.css"/><link rel="alternate" type="application/atom+xml" href="https://pulsar.apache.org/blog/atom.xml" title="Apache Pulsar Blog ATOM Feed"/><link rel="alternate" type="application/rss+xml" href="https://pulsar.apache.org/blog/feed.xml" title="Apache Pulsar Blog RSS Feed"/><link rel="stylesheet" href="/css/code-blocks-buttons.css"/><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js"></script><script type="text/javascript" src="/js/custom.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/en"><img class="logo" src="/img/pulsar.svg" alt="Apache Pulsar"/></a><a href="/en/versions"><h3>2.10.0</h3></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/en/getting-started-standalone" target="_self">Docs</a></li><li class=""><a href="/en/download" target="_self">Download</a></li><li class=""><a href="/docs/en/client-libraries" target="_self">Clients</a></li><li class=""><a href="#restapis" target="_self">REST APIs</a></li><li class=""><a href="#cli" target="_self">Cli</a></li><li class="siteNavGroupActive"><a href="/blog/" target="_self">Blog</a></li><li class=""><a href="#community" target="_self">Community</a></li><li class=""><a href="#apache" target="_self">Apache</a></li><li class=""><a href="https://pulsar-next.staged.apache.org/" target="_self">New Website (Beta)</a></li><span><li><a id="languages-menu" href="#"><img class="languages-icon" src="/img/language.svg" alt="Languages icon"/>English</a><div id="languages-dropdown" class="hide"><ul id="languages-dropdown-items"><li><a href="/ja">日本語</a></li><li><a href="/fr">Français</a></li><li><a href="/ko">한국어</a></li><li><a href="/zh-CN">中文</a></li><li><a href="/zh-TW">繁體中文</a></li><li><a href="https://crowdin.com/project/apache-pulsar" target="_blank" rel="noreferrer noopener">Help Translate</a></li></ul></div></li><script>
const languagesMenuItem = document.getElementById("languages-menu");
const languagesDropDown = document.getElementById("languages-dropdown");
languagesMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (languagesDropDown.className == "hide") {
languagesDropDown.className = "visible";
} else {
languagesDropDown.className = "hide";
}
});
</script></span></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Recent Posts</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Recent Posts</h3><ul class=""><li class="navListItem"><a class="navItem" href="/blog/2022/05/11/whats-new-in-pulsar-210">What’s New in Apache Pulsar 2.10</a></li><li class="navListItem"><a class="navItem" href="/blog/2022/05/11/apache-pulsar-community-welcomes-500th-contributor">The Apache Pulsar Community Welcomes 500th Contributor!</a></li><li class="navListItem"><a class="navItem" href="/blog/2022/04/08/Apache-Pulsar-2-9-2">What’s New in Apache Pulsar 2.9.2</a></li><li class="navListItem"><a class="navItem" href="/blog/2021/12/14/Apache-Pulsar-2-7-4">What’s New in Apache Pulsar 2.7.4</a></li><li class="navListItem"><a class="navItem" href="/blog/2021/12/11/Log4j-CVE">Log4j2 Zero Day vulnerability (CVE-2021-44228)</a></li></ul></div></div></section></div><script>
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
var links = coll[i].nextElementSibling.getElementsByTagName('*');
if (checkActiveCategory){
for (var j = 0; j < links.length; j++) {
if (links[j].classList.contains('navListItemActive')){
coll[i].nextElementSibling.classList.toggle('hide');
coll[i].childNodes[1].classList.toggle('rotate');
checkActiveCategory = false;
break;
}
}
}
coll[i].addEventListener('click', function() {
var arrow = this.childNodes[1];
arrow.classList.toggle('rotate');
var content = this.nextElementSibling;
content.classList.toggle('hide');
});
}
document.addEventListener('DOMContentLoaded', function() {
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
createToggler('#tocToggler', 'body', 'tocActive');
var headings = document.querySelector('.toc-headings');
headings && headings.addEventListener('click', function(event) {
var el = event.target;
while(el !== headings){
if (el.tagName === 'A') {
document.body.classList.remove('tocActive');
break;
} else{
el = el.parentNode;
}
}
}, false);
function createToggler(togglerSelector, targetSelector, className) {
var toggler = document.querySelector(togglerSelector);
var target = document.querySelector(targetSelector);
if (!toggler) {
return;
}
toggler.onclick = function(event) {
event.preventDefault();
target.classList.toggle(className);
};
}
});
</script></nav></div><div class="container mainContainer postContainer blogContainer"><div class="wrapper"><div class="lonePost"><div class="post"><header class="postHeader"><h1 class="postHeaderTitle"><a href="/blog/2019/02/20/Apache-Pulsar-2-3-0">Apache Pulsar 2.3.0</a></h1><p class="post-meta">February 20, 2019</p><div class="authorBlock"><p class="post-authorName"><a href="https://twitter.com/merlimat" target="_blank" rel="noreferrer noopener">Matteo Merli</a></p></div></header><div><span><p>The Apache Pulsar PMC is happy to announce the release of Pulsar 2.3.0. This
is the result of huge effort from the community, with over 480 commits and
a long list of new features, general improvements and bug fixes.</p>
<p>These improvements have been across the board in all of Pulsar components,
from new messaging features, to improved usability for Pulsar Functions
and Pulsar IO.</p>
<p>Check out the official <b><a href="/release-notes/#2.3.0">release notes</a></b> for a
detailed list of the changes, with links to the relevant pull-requests,
discussions and documentation.</p>
<p>Regarding new features introduced, I just want to highlight here a tiny
subset of them:</p>
<!--truncate-->
<h3><a class="anchor" aria-hidden="true" id="pulsar-functions-in-kubernetes"></a><a href="#pulsar-functions-in-kubernetes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Pulsar functions in Kubernetes</h3>
<p>It's now possible to use Kubernetes as the scheduler for Pulsar Functions.</p>
<p>When a Pulsar cluster is configured to use Kubernetes, submitting a
function, using CLI tools or REST API, will cause the function instances
to be submitted as Kubernetes pods rather than running as processes
or threads within the Pulsar functions worker.</p>
<p>With this runtime manager, it's possible to set quota on CPU/Mem and
have Kubernetes assign the required resources and enforce isolation
between different instances and functions.</p>
<h3><a class="anchor" aria-hidden="true" id="new-pulsar-io-connectors"></a><a href="#new-pulsar-io-connectors" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>New Pulsar IO connectors:</h3>
<p>A new batch of connectors was added, including MongoDB, Elastic Search,
HBase and local files source and sink.</p>
<p>We introduce support for doing
<a href="https://en.wikipedia.org/wiki/Change_data_capture">Change-Data-Capture</a>
with <a href="https://debezium.io/">Debezium</a>. This allows to record all
the update from a database into a Pulsar topic and use it for replication,
streaming jobs, cache updating, etc..</p>
<p>With Pulsar IO, Debezium will run as a regular Pulsar IO source,
completely managed by Pulsar. Users can easily submit a Debezium
builtin connector to a Pulsar cluster and start feeding data
from a long list of supported databases like MySQL, MongoDB,
PostgreSQL, Oracle and SQL Server.</p>
<p>Check out the <a href="/docs/io-cdc">Debezium connector</a> documentation for how
to get started in capturing database changes.</p>
<h3><a class="anchor" aria-hidden="true" id="token-authentication"></a><a href="#token-authentication" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Token Authentication</h3>
<p>Token Authentication provides a very simple and secure method of authentication for Pulsar.
This is based on <a href="https://jwt.io/">JSON Web Tokens</a>.</p>
<p>With tokens authentication, a client only needs to provide a single credential, or &quot;token&quot;, in the
form of an opaque string provided by either the system administrator or some automated service.</p>
<p>The Java code for a client using token authentication will look like:</p>
<pre><code class="hljs css language-java">PulsarClient client = PulsarClient.builder()
.serviceUrl(<span class="hljs-string">"pulsar://broker.example.com:6650/"</span>)
.authentication(
AuthenticationFactory.token(<span class="hljs-string">"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY"</span>)
.build();
</code></pre>
<p>See <a href="/docs/security-token-client">Client Authentication using tokens</a> for a complete walk through
and instructions on how to set it up and manage it.</p>
<h3><a class="anchor" aria-hidden="true" id="schema-support-in-python-client-library"></a><a href="#schema-support-in-python-client-library" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Schema support in Python client library</h3>
<p>This feature adds a Python idiomatic way to declare the schema
of a producer or consumer and integrates directly with the Pulsar
schema registry.</p>
<pre><code class="hljs css language-python"><span class="hljs-keyword">import</span> pulsar
<span class="hljs-keyword">from</span> pulsar.schema <span class="hljs-keyword">import</span> *
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Example</span><span class="hljs-params">(Record)</span>:</span>
a = String()
b = Integer()
c = Boolean()
producer = client.create_producer(
topic=<span class="hljs-string">'my-topic'</span>,
schema=AvroSchema(Example) )
producer.send(Example(a=<span class="hljs-string">'Hello'</span>, b=<span class="hljs-number">1</span>))
</code></pre>
<p>The above example will make the producer <code>Example</code> schema to be
validated by broker when we try to publish on <code>my-topic</code>. If the
topic has a schema that is incompatible, the producer creation will
fail.</p>
<p>Currently, the Python schema support Avro and JSON, in addition to
regular types like <code>str</code> and <code>bytes</code>.</p>
<p>The complete documentation is available at <a href="/docs/client-libraries-python/#schema">Python schema</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="function-state-api-in-python"></a><a href="#function-state-api-in-python" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Function state API in Python</h3>
<p>From 2.3.0, Python function can access the state in as similar
way as Java functions, through the context object.</p>
<pre><code class="hljs css language-python"><span class="hljs-keyword">import</span> pulsar
<span class="hljs-comment"># The classic ExclamationFunction that appends an</span>
<span class="hljs-comment"># exclamation at the end of the input</span>
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">WordCountFunction</span><span class="hljs-params">(pulsar.Function)</span>:</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process</span><span class="hljs-params">(self, input, context)</span>:</span>
<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> input.split():
context.incr_counter(word, <span class="hljs-number">1</span>)
<span class="hljs-keyword">return</span> input + <span class="hljs-string">"!"</span>
</code></pre>
<p>Available methods for state management in the context object are:</p>
<pre><code class="hljs css language-python"><span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">incr_counter</span><span class="hljs-params">(self, key, amount)</span>:</span>
<span class="hljs-string">"""incr the counter of a given key in the managed state"""</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_counter</span><span class="hljs-params">(self, key)</span>:</span>
<span class="hljs-string">"""get the counter of a given key in the managed state"""</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">put_state</span><span class="hljs-params">(self, key, value)</span>:</span>
<span class="hljs-string">"""update the value of a given key in the managed state"""</span>
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_state</span><span class="hljs-params">(self, key)</span>:</span>
<span class="hljs-string">"""get the value of a given key in the managed state"""</span>
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="conclusion"></a><a href="#conclusion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Conclusion</h2>
<p>Please <a href="/download">download</a> Pulsar 2.3.0 and report feedback, issues or any comment into our mailing lists,
slack channel or Github page. (<a href="/contact">Contact page</a>)</p>
</span></div></div><div class="blogSocialSection"><div class="blogSocialSectionItem"><a href="https://twitter.com/share" class="twitter-share-button" data-text="Apache Pulsar 2.3.0" data-url="https://pulsar.apache.org/blog/2019/02/20/Apache-Pulsar-2-3-0" data-related="true" data-show-count="false">Tweet</a></div></div></div><div class="blog-recent"><a class="button" href="/blog/">Recent Posts</a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#conclusion">Conclusion</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="copyright">Copyright © 2022 The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.</section><span><script>
const community = document.querySelector("a[href='#community']").parentNode;
const communityMenu =
'<li>' +
'<a id="community-menu" href="#">Community <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
'<div id="community-dropdown" class="hide">' +
'<ul id="community-dropdown-items">' +
'<li><a href="/en/contact">Contact</a></li>' +
'<li><a href="/en/contributing">Contributing</a></li>' +
'<li><a href="/en/coding-guide">Coding guide</a></li>' +
'<li><a href="/en/events">Events</a></li>' +
'<li><a href="https://twitter.com/Apache_Pulsar" target="_blank">Twitter &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/wiki" target="_blank">Wiki &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/issues" target="_blank">Issue tracking &#x2750</a></li>' +
'<li><a href="https://pulsar-summit.org/" target="_blank">Pulsar Summit &#x2750</a></li>' +
'<li>&nbsp;</li>' +
'<li><a href="/en/resources">Resources</a></li>' +
'<li><a href="/en/team">Team</a></li>' +
'<li><a href="/en/powered-by">Powered By</a></li>' +
'</ul>' +
'</div>' +
'</li>';
community.innerHTML = communityMenu;
const communityMenuItem = document.getElementById("community-menu");
const communityDropDown = document.getElementById("community-dropdown");
communityMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (communityDropDown.className == 'hide') {
communityDropDown.className = 'visible';
} else {
communityDropDown.className = 'hide';
}
});
</script></span></footer></div><script>window.twttr=(function(d,s, id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return t;js=d.createElement(s);js.id=id;js.src='https://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js, fjs);t._e = [];t.ready = function(f) {t._e.push(f);};return t;}(document, 'script', 'twitter-wjs'));</script></body></html>