Google Git
Sign in
apache / pulsar-site / aee2d9089b6b8e327f04261353077211f4abd636 / . / content / blog / 2020 / 04 / 23 / Apache-Pulsar-2-5-1.html
blob: 76124d40796d6718b597a41e5f05b011eabcb489 [file] [log] [blame]
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Apache Pulsar 2.5.1 · Apache Pulsar</title><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta name="generator" content="Docusaurus"/><meta name="description" content="We are proud to publish Apache Pulsar 2.5.1. This is the result of a huge effort from the community, with over 130 commits and a long list of new features, general improvements and bug fixes."/><meta name="docsearch:language" content="en"/><meta property="og:title" content="Apache Pulsar 2.5.1 · Apache Pulsar"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pulsar.apache.org/blog/2020/04/23/Apache-Pulsar-2-5-1"/><meta property="og:description" content="We are proud to publish Apache Pulsar 2.5.1. This is the result of a huge effort from the community, with over 130 commits and a long list of new features, general improvements and bug fixes."/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pulsar.apache.org/img/pulsar.svg"/><link rel="shortcut icon" href="/img/pulsar.ico"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/atom-one-dark.min.css"/><link rel="alternate" type="application/atom+xml" href="https://pulsar.apache.org/blog/atom.xml" title="Apache Pulsar Blog ATOM Feed"/><link rel="alternate" type="application/rss+xml" href="https://pulsar.apache.org/blog/feed.xml" title="Apache Pulsar Blog RSS Feed"/><link rel="stylesheet" href="/css/code-blocks-buttons.css"/><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js"></script><script type="text/javascript" src="/js/custom.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/en"><img class="logo" src="/img/pulsar.svg" alt="Apache Pulsar"/></a><a href="/en/versions"><h3>2.10.0</h3></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/en/getting-started-standalone" target="_self">Docs</a></li><li class=""><a href="/en/download" target="_self">Download</a></li><li class=""><a href="/docs/en/client-libraries" target="_self">Clients</a></li><li class=""><a href="#restapis" target="_self">REST APIs</a></li><li class=""><a href="#cli" target="_self">Cli</a></li><li class="siteNavGroupActive"><a href="/blog/" target="_self">Blog</a></li><li class=""><a href="#community" target="_self">Community</a></li><li class=""><a href="#apache" target="_self">Apache</a></li><li class=""><a href="https://pulsar-next.staged.apache.org/" target="_self">New Website (Beta)</a></li><span><li><a id="languages-menu" href="#"><img class="languages-icon" src="/img/language.svg" alt="Languages icon"/>English</a><div id="languages-dropdown" class="hide"><ul id="languages-dropdown-items"><li><a href="/ja">日本語</a></li><li><a href="/fr">Français</a></li><li><a href="/ko">한국어</a></li><li><a href="/zh-CN">中文</a></li><li><a href="/zh-TW">繁體中文</a></li><li><a href="https://crowdin.com/project/apache-pulsar" target="_blank" rel="noreferrer noopener">Help Translate</a></li></ul></div></li><script>
const languagesMenuItem = document.getElementById("languages-menu");
const languagesDropDown = document.getElementById("languages-dropdown");
languagesMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (languagesDropDown.className == "hide") {
languagesDropDown.className = "visible";
} else {
languagesDropDown.className = "hide";
}
});
</script></span></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Recent Posts</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Recent Posts</h3><ul class=""><li class="navListItem"><a class="navItem" href="/blog/2022/05/11/whats-new-in-pulsar-210">What’s New in Apache Pulsar 2.10</a></li><li class="navListItem"><a class="navItem" href="/blog/2022/05/11/apache-pulsar-community-welcomes-500th-contributor">The Apache Pulsar Community Welcomes 500th Contributor!</a></li><li class="navListItem"><a class="navItem" href="/blog/2022/04/08/Apache-Pulsar-2-9-2">What’s New in Apache Pulsar 2.9.2</a></li><li class="navListItem"><a class="navItem" href="/blog/2021/12/14/Apache-Pulsar-2-7-4">What’s New in Apache Pulsar 2.7.4</a></li><li class="navListItem"><a class="navItem" href="/blog/2021/12/11/Log4j-CVE">Log4j2 Zero Day vulnerability (CVE-2021-44228)</a></li></ul></div></div></section></div><script>
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
var links = coll[i].nextElementSibling.getElementsByTagName('*');
if (checkActiveCategory){
for (var j = 0; j < links.length; j++) {
if (links[j].classList.contains('navListItemActive')){
coll[i].nextElementSibling.classList.toggle('hide');
coll[i].childNodes[1].classList.toggle('rotate');
checkActiveCategory = false;
break;
}
}
}
coll[i].addEventListener('click', function() {
var arrow = this.childNodes[1];
arrow.classList.toggle('rotate');
var content = this.nextElementSibling;
content.classList.toggle('hide');
});
}
document.addEventListener('DOMContentLoaded', function() {
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
createToggler('#tocToggler', 'body', 'tocActive');
var headings = document.querySelector('.toc-headings');
headings && headings.addEventListener('click', function(event) {
var el = event.target;
while(el !== headings){
if (el.tagName === 'A') {
document.body.classList.remove('tocActive');
break;
} else{
el = el.parentNode;
}
}
}, false);
function createToggler(togglerSelector, targetSelector, className) {
var toggler = document.querySelector(togglerSelector);
var target = document.querySelector(targetSelector);
if (!toggler) {
return;
}
toggler.onclick = function(event) {
event.preventDefault();
target.classList.toggle(className);
};
}
});
</script></nav></div><div class="container mainContainer postContainer blogContainer"><div class="wrapper"><div class="lonePost"><div class="post"><header class="postHeader"><h1 class="postHeaderTitle"><a href="/blog/2020/04/23/Apache-Pulsar-2-5-1">Apache Pulsar 2.5.1</a></h1><p class="post-meta">April 23, 2020</p><div class="authorBlock"><p class="post-authorName"><a href="https://twitter.com/tuteng3" target="_blank" rel="noreferrer noopener">Guangning E</a></p></div></header><div><span><p>We are proud to publish Apache Pulsar 2.5.1. This is the result of a huge effort from the community, with over 130 commits and a long list of new features, general improvements and bug fixes.</p>
<p>For detailed changes related to 2.5.1 release, refer to the <b><a href="/release-notes/#2.5.1">release notes</a></b> and the <b><a href="https://github.com/apache/pulsar/pulls?q=is:pr%20label:release/2.5.1%20is:closed">PR list for Pulsar 2.5.1</a></b>.</p>
<p>The following justs highlights a tiny subset of new features.</p>
<!--truncate-->
<h2><a class="anchor" aria-hidden="true" id="refresh-authentication-credentials"></a><a href="#refresh-authentication-credentials" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Refresh authentication credentials</h2>
<p>In Pulsar 2.5.1, two more methods are introduced in the single <code>AuthenticationState</code> interface credentials holder. This helps enhance the Pulsar authentication framework to support credentials that expire over time and need to be refreshed by forcing clients to re-authenticate.</p>
<p>Existing authentication plugins are unaffected. If a new plugin wants to support expiration, it just overrides the <code>isExpired()</code> method. The Pulsar broker ensures to periodically check the expiration status for the AuthenticationState of every <code>ServerCnx</code> object. You can also use the <code>authenticationRefreshCheckSeconds</code> setting to control the frequency of the expiration check.</p>
<h2><a class="anchor" aria-hidden="true" id="upgrade-avro-to-191"></a><a href="#upgrade-avro-to-191" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Upgrade Avro to 1.9.1</h2>
<p>The library used to handle logical datetime values has been changed from Joda-Time to JSR-310. For keeping forward compatibility, Pulsar java client uses Joda-Time conversion for logical datetime. To use JSR-310 conversion, you can enable it in the schema definition.</p>
<pre><code class="hljs css language-java">AvroSchema.of(SchemaDefinition.builder()
.withJSR310ConversionEnabled(<span class="hljs-keyword">true</span>)
.build()
</code></pre>
<blockquote>
<h4><a class="anchor" aria-hidden="true" id="note"></a><a href="#note" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Note</h4>
<p>By default, Avro 1.9.1 enables the JSR310 datetimes, which might introduce some regression problems if users use source codes generated by Avro compiler 1.8.x and the source codes contain datetimes fields. It is recommended to use Avro 1.9.x compiler to recompile.
And, Avro may remove the Joda time support in the future. This may also be deleted in Pulsar in the future.</p>
</blockquote>
<h2><a class="anchor" aria-hidden="true" id="support-unloading-all-partitions-of-a-partitioned-topic"></a><a href="#support-unloading-all-partitions-of-a-partitioned-topic" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Support unloading all partitions of a partitioned topic</h2>
<p>Before Pulsar 2.5.1, Pulsar supports unloading a non-partitioned topic or a partition of a partitioned topic. If there is a partitioned topic with too many partitions, users need to get all partitions and unload them one by one. In Pulsar 2.5.1, we support unloading all partitions of a partitioned topic.</p>
<h2><a class="anchor" aria-hidden="true" id="supports-evenly-distributing-topics-count-when-splitting-bundle"></a><a href="#supports-evenly-distributing-topics-count-when-splitting-bundle" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Supports evenly distributing topics count when splitting bundle</h2>
<p>In Pulsar 2.5.1, we introduce an option(<code>-balance-topic-count</code>) for bundle split. When setting this option to true, the given bundle is split into two parts and each part has the same amount of topics. In addition, we bring in a new Load Manager implementation named <code>org.apache.pulsar.broker.loadbalance.impl.BalanceTopicCountModularLoadManager</code>. The new Load Manager implementation splits the bundle with balance topics count.
You can enable this feature in the broker.conf:</p>
<pre><code class="hljs"><span class="hljs-attr">defaultNamespaceBundleSplitAlgorithm</span>=topic_count_equally_divide
</code></pre>
<p>If you use the Pulsar Admin to split a bundle, you can use following command to split bundle based on topics count:</p>
<pre><code class="hljs">bin/pulsar-admin namespaces split-bundle -b 0x00000000_0xffffffff --split-algorithm-name topic_count_equally_divide public<span class="hljs-built_in">/default
</span></code></pre>
<h2><a class="anchor" aria-hidden="true" id="support-keyvalue-schema-for-pulsar-sql"></a><a href="#support-keyvalue-schema-for-pulsar-sql" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Support KeyValue schema for Pulsar SQL</h2>
<p>Before Pulsar 2.5.1, Pulsar SQL cannot read the keyValue schema data. In Pulsar 2.5.1, we add the prefix <code>key.</code> for the key field name, add the prefix <code>value.</code> for the value field name. Therefore, Pulsar SQL can read the keyValue schema data.</p>
<h2><a class="anchor" aria-hidden="true" id="update-netty-version-to-4145final"></a><a href="#update-netty-version-to-4145final" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Update Netty version to <code>4.1.45.Final</code></h2>
<p>Netty 4.1.43 has a bug, which prevents it from using Linux native Epoll transport. This makes Pulsar brokers fail over to NioEventLoopGroup even when running on Linux. The bug is fixed in Netty <code>4.1.45.Final</code> .</p>
<h2><a class="anchor" aria-hidden="true" id="improve-key_shared-subscription-message-dispatching-performance"></a><a href="#improve-key_shared-subscription-message-dispatching-performance" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Improve Key_Shared subscription message dispatching performance</h2>
<p>In Pulsar 2.5.1, to improve Key_Shared subscription message dispatching performance, we make the following operations for saving CPU usage which can improve non-batched message dispatch performance:</p>
<ul>
<li>Reduce making hash for the message key.</li>
<li>Reduce the number of finding consumers for message keys..</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="add-joda-time-logical-type-conversion"></a><a href="#add-joda-time-logical-type-conversion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Add Joda time logical type conversion</h2>
<p>In Pulsar 2.5.1, Avro is upgraded to 1.9.x and the default time conversion is changed to JSR-310. For forwarding compatibility, we add the Joda time conversion in Pulsar 2.5.1 and enable it by default</p>
<h2><a class="anchor" aria-hidden="true" id="support-deleting-inactive-topic-when-subscriptions-caught-up"></a><a href="#support-deleting-inactive-topic-when-subscriptions-caught-up" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Support deleting inactive topic when subscriptions caught up</h2>
<p>Before Pulsar 2.5.1, Pulsar supported deleting inactive topics that have no active producers or subscriptions. In Pulsar 2.5.1, we expose inactive topic delete mode in <code>broker.conf</code> to delete inactive topics that have no active producers or consumers but all subscriptions of the topic are caught up. You can enable this feature in the broker.conf:</p>
<pre><code class="hljs"><span class="hljs-attr">brokerDeleteInactiveTopicsMode</span>=delete_when_subscriptions_caught_up
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="introduce-maxmessagepublishbuffersizeinmb-configuration-to-avoid-broker-oom"></a><a href="#introduce-maxmessagepublishbuffersizeinmb-configuration-to-avoid-broker-oom" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Introduce maxMessagePublishBufferSizeInMB configuration to avoid broker OOM</h2>
<p>Before Pulsar 2.5.1, if a broker has a smaller direct memory (e.g. 2G) and runs pulsar-perf to write messages, the broker becomes unstable. Because the broker reads messages from the channel automatically and the ByteBuf cannot be released until the entry is written to Bookie successfully or the timeout expires.
In Pulsar 2.5.1, we introduce the <code>maxMessagePublishBufferSizeInMB</code> configuration to avoid broker OOM (Out of Memory). If the processing message size exceeds this value, the broker stops reading data from the connection. When the available size is greater than half of the maxMessagePublishBufferSizeInMB, the broker starts automatically reading data from the connection. You can set up the publish buffer size in broker.conf:</p>
<pre><code class="hljs"><span class="hljs-comment"># Max memory size for broker handling messages sending from producers.</span>
<span class="hljs-comment"># If the processing message size exceed this value, broker will stop read data</span>
<span class="hljs-comment"># from the connection. The processing messages means messages are sends to broker</span>
<span class="hljs-comment"># but broker have not send response to client, usually waiting to write to bookies.</span>
<span class="hljs-comment"># It's shared across all the topics running in the same broker.</span>
<span class="hljs-comment"># Use -1 to disable the memory limitation. Default is 1/2 of direct memory.</span>
<span class="hljs-attr">maxMessagePublishBufferSizeInMB</span>=
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="support-bouncycastle-fips-provider"></a><a href="#support-bouncycastle-fips-provider" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Support BouncyCastle FIPS provider</h2>
<p>In Pulsar 2.5.1, Pulsar supports BC-FIPS (BouncyCastle FIPS) provider. Before Pulsar 2.5.1, Pulsar only supported BouncyCastle (BC) provider, and BC JARs are tied strongly into both the broker and the client code. Users fail to change from the BC provider to the BC-FIPS provider. This feature splits the BC dependency out into a separate module. Therefore, users can freely switch between the BC provider and the BC-FIPS provider.</p>
<h2><a class="anchor" aria-hidden="true" id="allow-tenant-admin-to-manage-subscription-permission"></a><a href="#allow-tenant-admin-to-manage-subscription-permission" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Allow tenant Admin to manage subscription permission</h2>
<p>In previous releases, we have added support to grant subscriber-permission to manage subscription based APIs. However, grant-subscription-permission API requires super-user access and it creates too much dependency on system-admin when many tenants want to grant subscription permission.
In Pulsar 2.5.1, through the Restful API or the Pulsar Admin, we allow each tenant Admin to manage subscription permission in order to reduce administrative efforts for super users.</p>
<h2><a class="anchor" aria-hidden="true" id="allow-to-enabledisable-delayed-delivery-for-messages-on-namespace"></a><a href="#allow-to-enabledisable-delayed-delivery-for-messages-on-namespace" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Allow to enable/disable delayed delivery for messages on namespace</h2>
<p>In Pulsar 2.5.1, we add the <code>set-delayed-delivery</code> and <code>set-delayed-delivery-time</code> policies for the namespace. Therefore, Pulsar 2.5.1 allows to enable or disable delayed delayed delivery for messages on namespace.</p>
<h2><a class="anchor" aria-hidden="true" id="support-offloader-at-namespace-level"></a><a href="#support-offloader-at-namespace-level" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Support offloader at namespace level</h2>
<p>In previous releases, the offload operation only had the cluster-level configuration. Users cannot set the offload configuration at the namespace level. In Pulsar 2.5.1, we support using the Pulsar Admin to set the offloader at the namespace level.</p>
<h2><a class="anchor" aria-hidden="true" id="disallow-sub-auto-creation-by-admin-when-disabling-topic-auto-creation"></a><a href="#disallow-sub-auto-creation-by-admin-when-disabling-topic-auto-creation" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Disallow sub auto creation by Admin when disabling topic auto creation</h2>
<p>In previous releases, when Auto topic creation is disabled in KoP, non-partitioned topics are created with Flink Pulsar Source. To fix this bug, in Pulsar 2.5.1, we change the admin code to disable sub auto creation by the Admin when Auto topic creation is disabled.</p>
<h2><a class="anchor" aria-hidden="true" id="support-python-38-for-pulsar-client"></a><a href="#support-python-38-for-pulsar-client" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Support Python 3.8 for Pulsar client</h2>
<p>In pulsar 2.5.1, we add <code>3.8 cp38-cp38</code> to support Python 3.8 for the Pulsar client. Therefore, users can install the Pulsar client on Python 3.8 .</p>
<h2><a class="anchor" aria-hidden="true" id="provide-another-libpulsarwithdepsa-in-debianrpm-cpp-client-library"></a><a href="#provide-another-libpulsarwithdepsa-in-debianrpm-cpp-client-library" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Provide another <code>libpulsarwithdeps.a</code> in Debian/RPM cpp client library</h2>
<p>Pulsar 2.5.1 mainly provides 2 additional pulsar c++ client libraries in Debian/RPM:</p>
<ul>
<li>pulsarSharedNossl (libpulsarnossl.so): it is similar to pulsarShared(libpulsar.so), and has no SSL statically linked.</li>
<li>pulsarStaticWithDeps(libpulsarwithdeps.a): it is similar to pulsarStatic(libpulsar.a), and is archived in the dependencies libraries of <code>libboost_regex</code>, <code>libboost_system</code>, <code>libcurl</code>, <code>libprotobuf</code>, <code>libzstd</code> and <code>libz</code> statically.</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="reference"></a><a href="#reference" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Reference</h2>
<p>To download Apache Pulsar 2.5.1, click <a href="https://pulsar.apache.org/en/download/">here</a>.
If you have any questions or suggestions, contact us with mailing lists or slack.</p>
<ul>
<li><a href="mailto:users@pulsar.apache.org">users@pulsar.apache.org</a></li>
<li><a href="mailto:dev@pulsar.apache.org">dev@pulsar.apache.org</a></li>
<li>Pulsar slack channel: <a href="https://apache-pulsar.slack.com/">https://apache-pulsar.slack.com/</a></li>
<li>Self-registration at <a href="https://apache-pulsar.herokuapp.com/">https://apache-pulsar.herokuapp.com/</a>
Looking forward to your contributions to <a href="https://github.com/apache/pulsar">Pulsar</a>.</li>
</ul>
</span></div></div><div class="blogSocialSection"><div class="blogSocialSectionItem"><a href="https://twitter.com/share" class="twitter-share-button" data-text="Apache Pulsar 2.5.1" data-url="https://pulsar.apache.org/blog/2020/04/23/Apache-Pulsar-2-5-1" data-related="true" data-show-count="false">Tweet</a></div></div></div><div class="blog-recent"><a class="button" href="/blog/">Recent Posts</a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#refresh-authentication-credentials">Refresh authentication credentials</a></li><li><a href="#upgrade-avro-to-191">Upgrade Avro to 1.9.1</a></li><li><a href="#support-unloading-all-partitions-of-a-partitioned-topic">Support unloading all partitions of a partitioned topic</a></li><li><a href="#supports-evenly-distributing-topics-count-when-splitting-bundle">Supports evenly distributing topics count when splitting bundle</a></li><li><a href="#support-keyvalue-schema-for-pulsar-sql">Support KeyValue schema for Pulsar SQL</a></li><li><a href="#update-netty-version-to-4145final">Update Netty version to <code>4.1.45.Final</code></a></li><li><a href="#improve-key_shared-subscription-message-dispatching-performance">Improve Key_Shared subscription message dispatching performance</a></li><li><a href="#add-joda-time-logical-type-conversion">Add Joda time logical type conversion</a></li><li><a href="#support-deleting-inactive-topic-when-subscriptions-caught-up">Support deleting inactive topic when subscriptions caught up</a></li><li><a href="#introduce-maxmessagepublishbuffersizeinmb-configuration-to-avoid-broker-oom">Introduce maxMessagePublishBufferSizeInMB configuration to avoid broker OOM</a></li><li><a href="#support-bouncycastle-fips-provider">Support BouncyCastle FIPS provider</a></li><li><a href="#allow-tenant-admin-to-manage-subscription-permission">Allow tenant Admin to manage subscription permission</a></li><li><a href="#allow-to-enabledisable-delayed-delivery-for-messages-on-namespace">Allow to enable/disable delayed delivery for messages on namespace</a></li><li><a href="#support-offloader-at-namespace-level">Support offloader at namespace level</a></li><li><a href="#disallow-sub-auto-creation-by-admin-when-disabling-topic-auto-creation">Disallow sub auto creation by Admin when disabling topic auto creation</a></li><li><a href="#support-python-38-for-pulsar-client">Support Python 3.8 for Pulsar client</a></li><li><a href="#provide-another-libpulsarwithdepsa-in-debianrpm-cpp-client-library">Provide another <code>libpulsarwithdeps.a</code> in Debian/RPM cpp client library</a></li><li><a href="#reference">Reference</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="copyright">Copyright © 2022 The Apache Software Foundation. All Rights Reserved. Apache, Apache Pulsar and the Apache feather logo are trademarks of The Apache Software Foundation.</section><span><script>
const community = document.querySelector("a[href='#community']").parentNode;
const communityMenu =
'<li>' +
'<a id="community-menu" href="#">Community <span style="font-size: 0.75em">&nbsp;▼</span></a>' +
'<div id="community-dropdown" class="hide">' +
'<ul id="community-dropdown-items">' +
'<li><a href="/en/contact">Contact</a></li>' +
'<li><a href="/en/contributing">Contributing</a></li>' +
'<li><a href="/en/coding-guide">Coding guide</a></li>' +
'<li><a href="/en/events">Events</a></li>' +
'<li><a href="https://twitter.com/Apache_Pulsar" target="_blank">Twitter &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/wiki" target="_blank">Wiki &#x2750</a></li>' +
'<li><a href="https://github.com/apache/pulsar/issues" target="_blank">Issue tracking &#x2750</a></li>' +
'<li><a href="https://pulsar-summit.org/" target="_blank">Pulsar Summit &#x2750</a></li>' +
'<li>&nbsp;</li>' +
'<li><a href="/en/resources">Resources</a></li>' +
'<li><a href="/en/team">Team</a></li>' +
'<li><a href="/en/powered-by">Powered By</a></li>' +
'</ul>' +
'</div>' +
'</li>';
community.innerHTML = communityMenu;
const communityMenuItem = document.getElementById("community-menu");
const communityDropDown = document.getElementById("community-dropdown");
communityMenuItem.addEventListener("click", function(event) {
event.preventDefault();
if (communityDropDown.className == 'hide') {
communityDropDown.className = 'visible';
} else {
communityDropDown.className = 'hide';
}
});
</script></span></footer></div><script>window.twttr=(function(d,s, id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return t;js=d.createElement(s);js.id=id;js.src='https://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js, fjs);t._e = [];t.ready = function(f) {t._e.push(f);};return t;}(document, 'script', 'twitter-wjs'));</script></body></html>