| <!doctype html> |
| <html lang="en" dir="ltr" class="blog-wrapper blog-post-page plugin-blog plugin-id-default"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.0"> |
| <title data-rh="true">Apache Pulsar 2.5.2 | Apache Pulsar</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" name="twitter:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" property="og:url" content="https://pulsar.apache.org/blog/2020/05/19/Apache-Pulsar-2-5-2/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docsearch:docusaurus_tag" content="default"><meta data-rh="true" property="og:title" content="Apache Pulsar 2.5.2 | Apache Pulsar"><meta data-rh="true" name="description" content="We are proud to publish Apache Pulsar 2.5.2. This is the result of a huge effort from the community, with over 56 commits, general improvements and bug fixes."><meta data-rh="true" property="og:description" content="We are proud to publish Apache Pulsar 2.5.2. This is the result of a huge effort from the community, with over 56 commits, general improvements and bug fixes."><meta data-rh="true" property="og:type" content="article"><meta data-rh="true" property="article:published_time" content="2020-05-19T00:00:00.000Z"><meta data-rh="true" property="article:author" content="https://twitter.com/Jia_Zhai"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://pulsar.apache.org/blog/2020/05/19/Apache-Pulsar-2-5-2/"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/blog/2020/05/19/Apache-Pulsar-2-5-2/" hreflang="en"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/blog/2020/05/19/Apache-Pulsar-2-5-2/" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://WK2YL0SALL-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Pulsar RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Pulsar Atom Feed"> |
| |
| |
| |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache Pulsar" href="/opensearch.xml"> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="/css/katex-0.13.24.min.css" media="print" onload="this.media='all'"> |
| <script src="/js/sine-waves.min.js" async></script> |
| <script src="/js/matomo-agent.js"></script><link rel="stylesheet" href="/assets/css/styles.b0f65ef3.css"> |
| <link rel="preload" href="/assets/js/runtime~main.1d0ed2a7.js" as="script"> |
| <link rel="preload" href="/assets/js/main.e07a0c68.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:#282826;color:#fff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"> |
| <a class="announcement-bar" href="https://registration.socio.events/e/pulsarvirtualsummiteurope2024" target="_blank"> |
| <div class="announcement-bar__content"> |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 33 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <path d="M6.5 19.6001H16.1L15.3 29.2001L26.5 12.4H17.06L18.1 2.80005L6.5 19.6001Z" stroke="#F7F7F7" stroke-width="1.5" stroke-linejoin="round"/> |
| </svg> |
| |
| </svg> |
| |
| <span> |
| Get your free pass for Pulsar Virtual Summit Europe 2024 on May 14, 2024 🗓️ |
| </span> |
| |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <rect width="20" height="20" transform="translate(6 6)" fill="white" fill-opacity="0.01"/> |
| <path d="M17.6667 10.1667L23.5 16.0001M23.5 16.0001L17.6667 21.8334M23.5 16.0001L8.5 16.0001" stroke="white" stroke-linecap="round" stroke-linejoin="round"/> |
| <rect x="0.5" y="0.5" width="31" height="31" rx="15.5" stroke="white"/> |
| </svg> |
| |
| </svg> |
| </div> |
| </a> |
| </div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--light_HNdA" height="25" width="127"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--dark_i4oU" height="25" width="127"></div><b class="navbar__title text--truncate"></b></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Get Started</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/3.2.x/concepts-overview/">Concepts</a></li><li><a class="dropdown__link" href="/docs/3.2.x/">Quickstart</a></li><li><a class="dropdown__link" href="/ecosystem/">Ecosystem</a></li></ul></div><a class="navbar__item navbar__link" href="/docs/3.2.x/">Docs</a><a class="navbar__item navbar__link" href="/features/">Features</a><a class="navbar__item navbar__link" href="/use-cases/">Use Cases</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link community-dropdown">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link scroll-link scroll-welcome" id="scroll-welcome" href="/community/">Welcome</a></li><li><a class="dropdown__link scroll-link scroll-discussions" id="scroll-discussions" href="/community/#section-discussions">Discussions</a></li><li><a class="dropdown__link scroll-link" id="scroll-governance" href="/community/#section-governance">Governance</a></li><li><a class="dropdown__link scroll-link" id="scroll-community" href="/community/#section-community">Meet the Community</a></li><li><a class="dropdown__link scroll-link" id="scroll-contribute" href="/community/#section-contribute">Contribute</a></li><li><a class="dropdown__link" href="/contribute/">Contribution Guide</a></li><li><a href="https://github.com/apache/pulsar/wiki" target="_blank" rel="noopener noreferrer" class="dropdown__link">Wiki</a></li><li><a href="https://github.com/apache/pulsar/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Issue Tracking</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/blog/">Blog</a></li><li><a class="dropdown__link" href="/books/">Books</a></li><li><a class="dropdown__link" href="/case-studies/">Case Studies</a></li><li><a class="dropdown__link" href="/articles/">Articles</a></li><li><a class="dropdown__link" href="/presentations/">Presentations</a></li><li><a class="dropdown__link" href="/events/">Events</a></li></ul></div></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link navbar_download_button" href="/download/">Download</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-top--lg"><div class="row"><main class="col col--12" itemscope="" itemtype="https://schema.org/Blog"><article class="BlogContainerPost_N_n1" itemprop="blogPost" itemscope="" itemtype="https://schema.org/BlogPosting"><header><div><h1 class="title_xvU1" itemprop="headline">Apache Pulsar 2.5.2</h1><div class="row"><div class="col authorCol_q4o9"><div class="avatar"><div class="avatar__intro" itemprop="author" itemscope="" itemtype="https://schema.org/Person"><div class="avatar__name"><a href="https://twitter.com/Jia_Zhai" target="_blank" rel="noopener noreferrer" itemprop="url"><span itemprop="name">Jia Zhai</span></a></div></div></div></div></div><div class="container_iJTo"><time datetime="2020-05-19T00:00:00.000Z" itemprop="datePublished">May 19, 2020</time> · <!-- -->8 min read</div></div></header><div id="post-content" class="markdown" itemprop="articleBody"><p>We are proud to publish Apache Pulsar 2.5.2. This is the result of a huge effort from the community, with over 56 commits, general improvements and bug fixes.</p><p>For detailed changes related to 2.5.2 release, refer to the <b><a href="/release-notes/#2.5.2">release notes</a></b> and the <b><a href="https://github.com/apache/pulsar/pulls?q=is:pr%20label:release/2.5.2%20is:closed" target="_blank" rel="noopener noreferrer">PR list for Pulsar 2.5.2</a></b>.</p><p>The following highlights some improved features and fixed bugs in this release.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="implement-autotopiccreation-by-namespace-level-override">Implement AutoTopicCreation by namespace level override<a href="#implement-autotopiccreation-by-namespace-level-override" class="hash-link" aria-label="Direct link to Implement AutoTopicCreation by namespace level override" title="Direct link to Implement AutoTopicCreation by namespace level override"></a></h2><p>Introduce a new namespace policy <code>autoTopicCreationOverride</code>, which enables an override of broker <code>autoTopicCreation</code> settings on the namespace level. You can disable <code>autoTopicCreation</code> for the broker while allowing it on a specific namespace.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="add-customized-deletionlag-and-threshold-for-offloading-policies-per-namespace">Add customized deletionLag and threshold for offloading policies per namespace<a href="#add-customized-deletionlag-and-threshold-for-offloading-policies-per-namespace" class="hash-link" aria-label="Direct link to Add customized deletionLag and threshold for offloading policies per namespace" title="Direct link to Add customized deletionLag and threshold for offloading policies per namespace"></a></h2><p>Support configuring <code>deletionLag</code> and threshold in the offloading policy on the namespace level to remove data from the offloaded tiered storage.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="invalidate-managed-ledgers-zookeeper-cache-instead-of-reloading-on-watcher-triggered">Invalidate managed ledgers ZooKeeper cache instead of reloading on watcher triggered<a href="#invalidate-managed-ledgers-zookeeper-cache-instead-of-reloading-on-watcher-triggered" class="hash-link" aria-label="Direct link to Invalidate managed ledgers ZooKeeper cache instead of reloading on watcher triggered" title="Direct link to Invalidate managed ledgers ZooKeeper cache instead of reloading on watcher triggered"></a></h2><p>The ZooKeeper children cache is reloaded for z-nodes when topics are frequently created or deleted. This creates additional load on the ZooKeeper and the broker, slows down brokers and makes them less stable. In this release, <code>ZooKeeperManagedLedgerCache</code> is introduced to invalidate instead of reloading the ZooKeeper cache, when topics are created or deleted. This helps reduce pressures on the ZooKeeper.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="respect-retention-policy-when-there-is-no-traffic">Respect retention policy when there is no traffic<a href="#respect-retention-policy-when-there-is-no-traffic" class="hash-link" aria-label="Direct link to Respect retention policy when there is no traffic" title="Direct link to Respect retention policy when there is no traffic"></a></h2><p>In previous releases, retention is checked when the ledger rollover happens. So if the traffic is stopped, the ledgers are not cleaned up even if all the messages are already acknowledged. In Pulsar 2.5.2, <code>retentionCheckIntervalInSeconds</code> is introduced to check if consumed ledgers need to be trimmed between intervals. If the value is set to 0 or a negative number, the system does not check the consumed ledgers.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bump-netty-version-to-4148final">Bump Netty version to 4.1.48.Final<a href="#bump-netty-version-to-4148final" class="hash-link" aria-label="Direct link to Bump Netty version to 4.1.48.Final" title="Direct link to Bump Netty version to 4.1.48.Final"></a></h2><p>The ZlibDecoders in Netty 4.1.x (before 4.1.46) allow for unbounded memory allocation while decoding a ZlibEncoded byte stream. An attacker could send a large ZlibEncoded byte stream to the Netty server, forcing the server to allocate all of its free memory to a single decoder. The bug is fixed in Netty <code>4.1.48.Final</code> .</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="increase-timeout-for-loading-topics">Increase timeout for loading topics<a href="#increase-timeout-for-loading-topics" class="hash-link" aria-label="Direct link to Increase timeout for loading topics" title="Direct link to Increase timeout for loading topics"></a></h2><p>Loading replicated topics is quite an expensive operation and involves global ZooKeeper lookups and the start of many sub-processes. In Pulsar 2.5.2, we increase the timeout for loading topics which have many replicated clusters to 60 seconds.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-incorrect-cursor-state-for-cursor-without-consumers">Fix incorrect cursor state for cursor without consumers<a href="#fix-incorrect-cursor-state-for-cursor-without-consumers" class="hash-link" aria-label="Direct link to Fix incorrect cursor state for cursor without consumers" title="Direct link to Fix incorrect cursor state for cursor without consumers"></a></h2><p>If consumers of a subscription are closed, the cursor is set to inactive. But the cursor is set to active during <code>PulsarStats.updateStats()</code> when the backlog size is less than <code>backloggedCursorThresholdEntries</code>. In Pulsar 2.5.2, we move the <code>checkBackloggedCursors()</code> from <code>ManagedLedger</code> to <code>Topic</code> and check the consumer list to fix this bug.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="change-non-durable-cursor-to-active-to-improve-performance">Change non-durable cursor to active to improve performance<a href="#change-non-durable-cursor-to-active-to-improve-performance" class="hash-link" aria-label="Direct link to Change non-durable cursor to active to improve performance" title="Direct link to Change non-durable cursor to active to improve performance"></a></h2><p>In non-durable subscription mode, the cursor is not active, which leads to the written entries not being put into cache. This would degrade the reading performance. In Pulsar 2.5.2, we set the <code>NonDurableCursorImpl</code> to active and remove three override methods <code>setActive()</code>, <code>isActive()</code>, <code>setInactive()</code> to improve the reading performance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="add-keystore-configurations-to-tls">Add keystore configurations to TLS<a href="#add-keystore-configurations-to-tls" class="hash-link" aria-label="Direct link to Add keystore configurations to TLS" title="Direct link to Add keystore configurations to TLS"></a></h2><p>In Pulsar 2.5.2, we add keystore configurations to the TLS to allow users to define their own CA certificates while the internal communication uses an internal CA certificate. This change keeps the original TLS settings untouched, and adds new configurations in needed paths.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="close-producer-when-the-topic-does-not-exists">Close producer when the topic does not exists<a href="#close-producer-when-the-topic-does-not-exists" class="hash-link" aria-label="Direct link to Close producer when the topic does not exists" title="Direct link to Close producer when the topic does not exists"></a></h2><p>In previous releases, when we create a producer for a non-existent topic, the <code>ProducerImpl</code> object is hanging in the dump. This leads to OOM in micro-service which by mistake tries to produce consistently to a non-existent topic. In Pulsar 2.5.2, we fix the bug in the following two aspects:</p><ul><li>Fix the exception handle for a non-existent topic.</li><li>Change state to <code>Close</code> when the producer gets the <code>TopicDoesNotExists</code> exception.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-topicpublishratelimiter-not-effective-after-restarting-broker">Fix <code>topicPublishRateLimiter</code> not effective after restarting broker<a href="#fix-topicpublishratelimiter-not-effective-after-restarting-broker" class="hash-link" aria-label="Direct link to fix-topicpublishratelimiter-not-effective-after-restarting-broker" title="Direct link to fix-topicpublishratelimiter-not-effective-after-restarting-broker"></a></h2><p>In previous releases, when a publishing rate is configured on the namespace, it can limit the publishing rate. But when the broker is restarted, the limit expires. In Pulsar 2.5.2, this bug is fixed.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="expose-pulsar_out_bytes_total-and-pulsar_out_messages_total-for-namespacesubscriptionconsumer">Expose pulsar_out_bytes_total and pulsar_out_messages_total for namespace/subscription/consumer<a href="#expose-pulsar_out_bytes_total-and-pulsar_out_messages_total-for-namespacesubscriptionconsumer" class="hash-link" aria-label="Direct link to Expose pulsar_out_bytes_total and pulsar_out_messages_total for namespace/subscription/consumer" title="Direct link to Expose pulsar_out_bytes_total and pulsar_out_messages_total for namespace/subscription/consumer"></a></h2><p>Add pulsar_out_bytes_total and pulsar_out_messages_total for the namespace, subscription, and consumer. This helps to avoid missing the rate to be computed in Prometheus or missing change of rates within the scraping interval.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-ttldurationdefaultinseconds-policy">Fix <code>ttlDurationDefaultInSeconds</code> policy<a href="#fix-ttldurationdefaultinseconds-policy" class="hash-link" aria-label="Direct link to fix-ttldurationdefaultinseconds-policy" title="Direct link to fix-ttldurationdefaultinseconds-policy"></a></h2><p>The TTL for namespaces should be retrieved from the broker configuration if it is not configured at namespace policies. In previous releases, the code only returns the value stored in namespace policies directly without judging if the TTL is configured or not. In Pulsar 2.5.2, we add a condition to test if TTL is configured at namespace policies. If not, the broker retrieves value stored in broker configuration and returns it as the output.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-long-field-parse-in-genricjsonrecord">Fix long field parse in GenricJsonRecord<a href="#fix-long-field-parse-in-genricjsonrecord" class="hash-link" aria-label="Direct link to Fix long field parse in GenricJsonRecord" title="Direct link to Fix long field parse in GenricJsonRecord"></a></h2><p>For messages sent in JSON schema, the long field is decoded as int if its value is smaller than <code>Integer.MAX_VALUE</code>. Otherwise, the long field is decoded as a string. Pulsar 2.5.2 introduces a field type check in GenericJsonRecord to fix this bug.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-the-leak-of-cursor-reset-if-message-encode-fails-in-avro-schema">Fix the leak of cursor reset if message encode fails in Avro schema<a href="#fix-the-leak-of-cursor-reset-if-message-encode-fails-in-avro-schema" class="hash-link" aria-label="Direct link to Fix the leak of cursor reset if message encode fails in Avro schema" title="Direct link to Fix the leak of cursor reset if message encode fails in Avro schema"></a></h2><p>If the Avro encode for a message fails after a few bytes are written, the cursor in the stream is not reset. The following <code>flush()</code>, which normally resets the cursor, is skipped if there is an exception. In Pulsar 2.5.2, we introduced a <code>flush()</code> in the finally block to fix this bug.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="update-topic-partitions-automatically">Update topic partitions automatically<a href="#update-topic-partitions-automatically" class="hash-link" aria-label="Direct link to Update topic partitions automatically" title="Direct link to Update topic partitions automatically"></a></h2><p>In Pulsar 2.5.2, the C++ client supports previously-created producers and consumers to automatically update partitions when the partitions for a topic are updated.</p><ul><li>Add a <code>boost::asio::deadline_timer</code> to <code>PartitionedConsumerImpl</code> and <code>PartitionedProducerImpl</code> to register a lookup task to detect partition changes periodically.</li><li>Add an unsigned int configuration parameter to indicate the period of detecting partition changes.</li><li>Unlock the <code>mutex_</code> in <code>PartitionedConsumerImpl::receive</code> after <code>state_</code> were checked.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-default-message-id-in-sent-callback">Fix default message ID in sent callback<a href="#fix-default-message-id-in-sent-callback" class="hash-link" aria-label="Direct link to Fix default message ID in sent callback" title="Direct link to Fix default message ID in sent callback"></a></h2><p>In previous releases, the <code>MessageId</code> in the callback is always the default value (<code>-1, -1, -1, -1</code>). In Pulsar 2.5.2, we remove the useless field <code>messageId</code> of <code>BatchMessageContainer::MessageContainer</code> and add the <code>const MessageId&</code> argument to <code>batchMessageCallBack</code>. Therefore, we can get the correct message ID in the callback if the message is sent successfully.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-message-id-error-if-messages-are-sent-to-partitioned-topics">Fix message ID error if messages are sent to partitioned topics<a href="#fix-message-id-error-if-messages-are-sent-to-partitioned-topics" class="hash-link" aria-label="Direct link to Fix message ID error if messages are sent to partitioned topics" title="Direct link to Fix message ID error if messages are sent to partitioned topics"></a></h2><p>If messages are sent to a partitioned topic, the <code>partition</code> field of the message ID is always set to -1 because the <code>SendReceipt</code> command only contains the ledger ID and the entry ID. In Pulsar 2.5.2, we fix this bug by adding a <code>partition</code> field to <code>ProducerImpl</code> and setting the <code>partition</code> field of the message ID with it in the <code>ackReceived</code> method.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="support-async-mode-for-pulsar-functions">Support Async mode for Pulsar Functions<a href="#support-async-mode-for-pulsar-functions" class="hash-link" aria-label="Direct link to Support Async mode for Pulsar Functions" title="Direct link to Support Async mode for Pulsar Functions"></a></h2><p>In previous releases, Pulsar Functions does not support the Async mode, such as the user passed in a Function in the following format:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">Function<I, CompletableFuture<O>></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This kind of function is useful if the Pulsar Functions use RPCs to call external systems. Therefore, in Pulsar 2.5.2, we introduce Async mode support for Pulsar Functions.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-localrunner-netty-dependency-issue">Fix localrunner netty dependency issue<a href="#fix-localrunner-netty-dependency-issue" class="hash-link" aria-label="Direct link to Fix localrunner netty dependency issue" title="Direct link to Fix localrunner netty dependency issue"></a></h2><p>In Pulsar 2.5.2, we add a Log4j2 configuration file for pulsar-functions-local-runner to log to console by default. This helps troubleshoot the problem that Netty libraries are missing and the class is not found, when pulling in pulsar-functions-local-runner as a dependency and attempting to run Pulsar Functions locally.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fix-serde-validation-of-pulsar-functions-update">Fix SerDe validation of Pulsar Functions update<a href="#fix-serde-validation-of-pulsar-functions-update" class="hash-link" aria-label="Direct link to Fix SerDe validation of Pulsar Functions update" title="Direct link to Fix SerDe validation of Pulsar Functions update"></a></h2><p>In previous releases, the <code>outputSchemaType</code> field is improperly used to validate parameters for Pulsar Function updates. In fact, the <code>outputSerdeClassName</code> parameter should be used. In Pulsar 2.5.2, we fix this bug.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="avoid-pre-fetching-too-much-data-when-offloading-data-to-hdfs">Avoid pre-fetching too much data when offloading data to HDFS<a href="#avoid-pre-fetching-too-much-data-when-offloading-data-to-hdfs" class="hash-link" aria-label="Direct link to Avoid pre-fetching too much data when offloading data to HDFS" title="Direct link to Avoid pre-fetching too much data when offloading data to HDFS"></a></h2><p>If too much data is pre-fetched when data is offloaded to HDFS, it may cause severe OOM. In Pulsar 2.5.2, the <code>managedLedgerOffloadPrefetchRounds</code> is introduced, which is used to set the maximum pre-fetch rounds for ledger reading for offloading data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="jdbc-sink-handles-null-fields-in-schema">JDBC sink handles null fields in schema<a href="#jdbc-sink-handles-null-fields-in-schema" class="hash-link" aria-label="Direct link to JDBC sink handles null fields in schema" title="Direct link to JDBC sink handles null fields in schema"></a></h2><p>JDBC sink does not handle <code>null</code> fields. The schema registered in Pulsar allows for it and the table schema in MySQL has a column of the same name. When messages are sent to the JDBC sink without that field, an exception is thrown. In Pulsar 2.5.2, the JDBC sink uses the <code>setColumnNull</code> method to properly reflect the null field value in the database row.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reference">Reference<a href="#reference" class="hash-link" aria-label="Direct link to Reference" title="Direct link to Reference"></a></h2><p>To download Apache Pulsar 2.5.2, click <a href="https://pulsar.apache.org/download/" target="_blank" rel="noopener noreferrer">here</a>.</p><p>If you have any questions or suggestions, contact us with mailing lists or slack.</p><ul><li><a href="mailto:users@pulsar.apache.org" target="_blank" rel="noopener noreferrer">users@pulsar.apache.org</a></li><li><a href="mailto:dev@pulsar.apache.org" target="_blank" rel="noopener noreferrer">dev@pulsar.apache.org</a></li><li>Pulsar slack channel: <a href="https://apache-pulsar.slack.com/" target="_blank" rel="noopener noreferrer">https://apache-pulsar.slack.com/</a></li><li>Self-registration at <a href="https://apache-pulsar.herokuapp.com/" target="_blank" rel="noopener noreferrer">https://apache-pulsar.herokuapp.com/</a></li></ul><p>Looking forward to your contributions to <a href="https://github.com/apache/pulsar" target="_blank" rel="noopener noreferrer">Pulsar</a>.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_Wr5y"><div class="col margin-top--sm"><a href="https://github.com/apache/pulsar-site/edit/main/blog/2020-05-19-Apache-Pulsar-2-5-2.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div></footer></article><nav class="pagination-nav blog-post-pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><div><a class="pagination-nav__link pagination-nav__link--prev" href="/blog/2020/06/18/Apache-Pulsar-2-6-0/"><div class="pagination-nav__sublabel">Newer Post</div><div class="pagination-nav__label">Apache Pulsar 2.6.0</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/blog/2020/04/23/Apache-Pulsar-2-5-1/"><div class="pagination-nav__sublabel">Older Post</div><div class="pagination-nav__label">Apache Pulsar 2.5.1</div></a><div class="blog-post-pagination-nav-more"><a href="/blog">Show All</a></div></div></nav></main></div></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Foundation<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/events/current-event.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Events<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/security/">Security</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Privacy<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/contact/">Contact</a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"> |
| <div class="social-icons"> |
| <a target="_blank" href="https://communityinviter.com/apps/apache-pulsar/apache-pulsar" aria-label="Join the Apache Pulsar Slack workspace"> |
| <img alt="Slack logo" src="/img/slack-white.svg" width="26"> |
| </a> |
| <a target="_blank" href="https://github.com/apache/pulsar/" aria-label="View the Apache Pulsar project on GitHub"> |
| <img alt="GitHub logo" src="/img/github-white.svg" width="26"> |
| </a> |
| </div> |
| </li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a class="footerLogoLink_BH7S" href="/"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></a></div><div class="footer__copyright"> |
| <div> |
| <img class="footer-apache-logo" src="/img/feather-logo-white.svg" alt="" width="20"> |
| The Apache Software Foundation |
| </div> |
| <p>Apache Pulsar is available under the Apache License, version 2.0. Apache Pulsar is an open-source, distributed messaging and streaming platform built for the cloud.</p> |
| <p>Copyright © 2024 The Apache Software Foundation. All Rights Reserved. Apache, Pulsar, Apache Pulsar, and the Apache feather logo are trademarks or registered trademarks of The Apache Software Foundation.</p> |
| </div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.1d0ed2a7.js"></script> |
| <script src="/assets/js/main.e07a0c68.js"></script> |
| </body> |
| </html> |
