| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-3.0.x plugin-docs plugin-id-default docs-doc-id-developing-binary-protocol"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.0"> |
| <title data-rh="true">Pulsar binary protocol specification | Apache Pulsar</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" name="twitter:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" property="og:url" content="https://pulsar.apache.org/docs/3.0.x/developing-binary-protocol/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="3.0.x"><meta data-rh="true" name="docusaurus_tag" content="docs-default-3.0.x"><meta data-rh="true" name="docsearch:version" content="3.0.x"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-3.0.x"><meta data-rh="true" property="og:title" content="Pulsar binary protocol specification | Apache Pulsar"><meta data-rh="true" name="description" content="Pulsar uses a custom binary protocol for communications between producers/consumers and brokers. This protocol is designed to support required features, such as acknowledgments and flow control, while ensuring maximum transport and implementation efficiency."><meta data-rh="true" property="og:description" content="Pulsar uses a custom binary protocol for communications between producers/consumers and brokers. This protocol is designed to support required features, such as acknowledgments and flow control, while ensuring maximum transport and implementation efficiency."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/docs/3.0.x/developing-binary-protocol/" hreflang="en"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/docs/3.0.x/developing-binary-protocol/" hreflang="x-default"><link data-rh="true" rel="canonical" href="https://pulsar.apache.org/docs/developing-binary-protocol/"><link data-rh="true" rel="preconnect" href="https://WK2YL0SALL-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Pulsar RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Pulsar Atom Feed"> |
| |
| |
| |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache Pulsar" href="/opensearch.xml"> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="/css/katex-0.13.24.min.css" media="print" onload="this.media='all'"> |
| <script src="/js/sine-waves.min.js" async></script> |
| <script src="/js/matomo-agent.js"></script><link rel="stylesheet" href="/assets/css/styles.83bad5a1.css"> |
| <link rel="preload" href="/assets/js/runtime~main.f1ab6cb0.js" as="script"> |
| <link rel="preload" href="/assets/js/main.9801eec2.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:#282826;color:#fff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"> |
| <a class="announcement-bar" href="https://registration.socio.events/e/pulsarvirtualsummiteurope2024" target="_blank"> |
| <div class="announcement-bar__content"> |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 33 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <path d="M6.5 19.6001H16.1L15.3 29.2001L26.5 12.4H17.06L18.1 2.80005L6.5 19.6001Z" stroke="#F7F7F7" stroke-width="1.5" stroke-linejoin="round"/> |
| </svg> |
| |
| </svg> |
| |
| <span> |
| Get your free pass for Pulsar Virtual Summit Europe 2024 on May 14, 2024 🗓️ |
| </span> |
| |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <rect width="20" height="20" transform="translate(6 6)" fill="white" fill-opacity="0.01"/> |
| <path d="M17.6667 10.1667L23.5 16.0001M23.5 16.0001L17.6667 21.8334M23.5 16.0001L8.5 16.0001" stroke="white" stroke-linecap="round" stroke-linejoin="round"/> |
| <rect x="0.5" y="0.5" width="31" height="31" rx="15.5" stroke="white"/> |
| </svg> |
| |
| </svg> |
| </div> |
| </a> |
| </div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--light_HNdA" height="25" width="127"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--dark_i4oU" height="25" width="127"></div><b class="navbar__title text--truncate"></b></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Get Started</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/3.2.x/concepts-overview/">Concepts</a></li><li><a class="dropdown__link" href="/docs/3.2.x/">Quickstart</a></li><li><a class="dropdown__link" href="/ecosystem/">Ecosystem</a></li></ul></div><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs/3.0.x/">Docs</a><a class="navbar__item navbar__link" href="/features/">Features</a><a class="navbar__item navbar__link" href="/use-cases/">Use Cases</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link community-dropdown">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link scroll-link scroll-welcome" id="scroll-welcome" href="/community/">Welcome</a></li><li><a class="dropdown__link scroll-link scroll-discussions" id="scroll-discussions" href="/community/#section-discussions">Discussions</a></li><li><a class="dropdown__link scroll-link" id="scroll-governance" href="/community/#section-governance">Governance</a></li><li><a class="dropdown__link scroll-link" id="scroll-community" href="/community/#section-community">Meet the Community</a></li><li><a class="dropdown__link scroll-link" id="scroll-contribute" href="/community/#section-contribute">Contribute</a></li><li><a class="dropdown__link" href="/contribute/">Contribution Guide</a></li><li><a href="https://github.com/apache/pulsar/wiki" target="_blank" rel="noopener noreferrer" class="dropdown__link">Wiki</a></li><li><a href="https://github.com/apache/pulsar/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Issue Tracking</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/blog/">Blog</a></li><li><a class="dropdown__link" href="/books/">Books</a></li><li><a class="dropdown__link" href="/case-studies/">Case Studies</a></li><li><a class="dropdown__link" href="/articles/">Articles</a></li><li><a class="dropdown__link" href="/presentations/">Presentations</a></li><li><a class="dropdown__link" href="/events/">Events</a></li></ul></div></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link navbar_download_button" href="/download/">Download</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_mhZE"><div class="sidebarVersionSwitch_EHpo">Version:<div class="navbar__item dropdown dropdown--hoverable"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/3.0.x/">3.0.x</a><ul class="dropdown__menu"><li><a class="dropdown__link">Next</a></li><li><a class="dropdown__link">3.2.x</a></li><li><a class="dropdown__link">3.0.x LTS</a></li><li><a class="dropdown__link">Others</a></li></ul></div></div><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/3.0.x/">About</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.0.x/getting-started-home/">Get Started</a><button aria-label="Toggle the collapsible sidebar category 'Get Started'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/concepts-overview/">Concepts and Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/schema-overview/">Pulsar Schema</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/functions-overview/">Pulsar Functions</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/io-overview/">Pulsar IO</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/sql-overview/">Pulsar SQL</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/tiered-storage-overview/">Tiered Storage</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/txn-why/">Transactions</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.0.x/install-deploy-upgrade-landing/">Deployment</a><button aria-label="Toggle the collapsible sidebar category 'Deployment'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/administration-zk-bk/">Administration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/administration-stats/">Observability</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/security-overview/">Security</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/performance-pulsar-perf/">Performance</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/client-libraries/">Client Libraries</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/admin-api-overview/">Admin API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.0.x/adaptors-kafka/">Adaptors</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.0.x/how-to-landing/">Tutorials</a><button aria-label="Toggle the collapsible sidebar category 'Tutorials'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--active" aria-expanded="true" href="/docs/3.0.x/developers-landing/">Development</a><button aria-label="Toggle the collapsible sidebar category 'Development'" type="button" class="clean-btn menu__caret"></button></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.0.x/develop-tools/">Simulation tools</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/3.0.x/developing-binary-protocol/">Binary protocol</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.0.x/develop-load-manager/">Modular load manager</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.0.x/develop-plugin/">Plugin</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.0.x/reference-landing/">Reference</a><button aria-label="Toggle the collapsible sidebar category 'Reference'" type="button" class="clean-btn menu__caret"></button></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item"><a class="breadcrumbs__link" itemprop="item" href="/docs/3.0.x/developers-landing/"><span itemprop="name">Development</span></a><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Binary protocol</span><meta itemprop="position" content="2"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Pulsar binary protocol specification</h1></header><p>Pulsar uses a custom binary protocol for communications between producers/consumers and brokers. This protocol is designed to support required features, such as acknowledgments and flow control, while ensuring maximum transport and implementation efficiency.</p><p>Clients and brokers exchange <em>commands</em> with each other. Commands are formatted as binary <a href="https://developers.google.com/protocol-buffers/" target="_blank" rel="noopener noreferrer">protocol buffer</a> (aka <em>protobuf</em>) messages. The format of protobuf commands is specified in the <a href="https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/proto/PulsarApi.proto" target="_blank" rel="noopener noreferrer"><code>PulsarApi.proto</code></a> file and also documented in the <a href="#protobuf-interface">Protobuf interface</a> section below.</p><blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="connection-sharing">Connection sharing<a href="#connection-sharing" class="hash-link" aria-label="Direct link to Connection sharing" title="Direct link to Connection sharing"></a></h3><p>Commands for different producers and consumers can be interleaved and sent through the same connection without restriction.</p></blockquote><p>All commands associated with Pulsar's protocol are contained in a <a href="#pulsar.proto.BaseCommand"><code>BaseCommand</code></a> protobuf message that includes a <a href="#pulsar.proto.Type"><code>Type</code></a> <a href="https://developers.google.com/protocol-buffers/docs/proto#enum" target="_blank" rel="noopener noreferrer">enum</a> with all possible subcommands as optional fields. <code>BaseCommand</code> messages can specify only one subcommand.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="framing">Framing<a href="#framing" class="hash-link" aria-label="Direct link to Framing" title="Direct link to Framing"></a></h2><p>Since protobuf doesn't provide any sort of message frame, all messages in the Pulsar protocol are prepended with a 4-byte field that specifies the size of the frame. The maximum allowable size of a single frame is 5 MB.</p><p>The Pulsar protocol allows for two types of commands:</p><ol><li><strong>Simple commands</strong> that do not carry a message payload.</li><li><strong>Payload commands</strong> that bear a payload that is used when publishing or delivering messages. In payload commands, the protobuf command data is followed by protobuf <a href="#message-metadata">metadata</a> and then the payload, which is passed in a raw format outside of protobuf. All sizes are passed as 4-byte unsigned big-endian integers.</li></ol><blockquote><p>Message payloads are passed in raw format rather than protobuf format for efficiency reasons.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simple-commands">Simple commands<a href="#simple-commands" class="hash-link" aria-label="Direct link to Simple commands" title="Direct link to Simple commands"></a></h3><p>Simple (payload-free) commands have this basic structure:</p><table><thead><tr><th align="left">Component</th><th align="left">Description</th><th align="left">Size (in bytes)</th></tr></thead><tbody><tr><td align="left"><code>totalSize</code></td><td align="left">The size of the frame, counting everything that comes after it (in bytes)</td><td align="left">4</td></tr><tr><td align="left"><code>commandSize</code></td><td align="left">The size of the protobuf-serialized command</td><td align="left">4</td></tr><tr><td align="left"><code>command</code></td><td align="left">The protobuf serialized command</td><td align="left"></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="message-commands">Message commands<a href="#message-commands" class="hash-link" aria-label="Direct link to Message commands" title="Direct link to Message commands"></a></h3><p>Payload commands have this basic structure:</p><table><thead><tr><th align="left">Component</th><th align="left">Required or optional</th><th align="left">Description</th><th align="left">Size (in bytes)</th></tr></thead><tbody><tr><td align="left"><code>totalSize</code></td><td align="left">Required</td><td align="left">The size of the frame, counting everything that comes after it (in bytes)</td><td align="left">4</td></tr><tr><td align="left"><code>commandSize</code></td><td align="left">Required</td><td align="left">The size of the protobuf-serialized command</td><td align="left">4</td></tr><tr><td align="left"><code>command</code></td><td align="left">Required</td><td align="left">The protobuf serialized command</td><td align="left"></td></tr><tr><td align="left"><code>magicNumberOfBrokerEntryMetadata</code></td><td align="left">Optional</td><td align="left">A 2-byte byte array (<code>0x0e02</code>) identifying the broker entry metadata <br> <strong>Note</strong>: <code>magicNumberOfBrokerEntryMetadata</code> , <code>brokerEntryMetadataSize</code>, and <code>brokerEntryMetadata</code> should be used <strong>together</strong>.</td><td align="left">2</td></tr><tr><td align="left"><code>brokerEntryMetadataSize</code></td><td align="left">Optional</td><td align="left">The size of the broker entry metadata</td><td align="left">4</td></tr><tr><td align="left"><code>brokerEntryMetadata</code></td><td align="left">Optional</td><td align="left">The broker entry metadata stored as a binary protobuf message</td><td align="left"></td></tr><tr><td align="left"><code>magicNumber</code></td><td align="left">Required</td><td align="left">A 2-byte byte array (<code>0x0e01</code>) identifying the current format</td><td align="left">2</td></tr><tr><td align="left"><code>checksum</code></td><td align="left">Required</td><td align="left">A <a href="http://www.evanjones.ca/crc32c.html" target="_blank" rel="noopener noreferrer">CRC32-C checksum</a> of everything that comes after it</td><td align="left">4</td></tr><tr><td align="left"><code>metadataSize</code></td><td align="left">Required</td><td align="left">The size of the message <a href="#message-metadata">metadata</a></td><td align="left">4</td></tr><tr><td align="left"><code>metadata</code></td><td align="left">Required</td><td align="left">The message <a href="#message-metadata">metadata</a> stored as a binary protobuf message</td><td align="left"></td></tr><tr><td align="left"><code>payload</code></td><td align="left">Required</td><td align="left">Anything left in the frame is considered the payload and can include any sequence of bytes</td><td align="left"></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="broker-entry-metadata">Broker entry metadata<a href="#broker-entry-metadata" class="hash-link" aria-label="Direct link to Broker entry metadata" title="Direct link to Broker entry metadata"></a></h2><p>Broker entry metadata is stored alongside the message metadata as a serialized protobuf message. |
| It is created by the broker when the message arrived at the broker and passed without changes to the consumer if configured.</p><table><thead><tr><th align="left">Field</th><th align="left">Required or optional</th><th align="left">Description</th></tr></thead><tbody><tr><td align="left"><code>broker_timestamp</code></td><td align="left">Optional</td><td align="left">The timestamp when a message arrived at the broker (<code>id est</code> as the number of milliseconds since January 1st, 1970 in UTC)</td></tr><tr><td align="left"><code>index</code></td><td align="left">Optional</td><td align="left">The index of the message. It is assigned by the broker.</td></tr></tbody></table><p>If you want to use broker entry metadata for <strong>brokers</strong>, configure the <a href="/docs/3.0.x/reference-configuration/"><code>brokerEntryMetadataInterceptors</code></a> parameter in the <code>broker.conf</code> file.</p><p>If you want to use broker entry metadata for <strong>consumers</strong>:</p><ol><li><p>Use the client protocol version <a href="https://github.com/apache/pulsar/blob/ca37e67211feda4f7e0984e6414e707f1c1dfd07/pulsar-common/src/main/proto/PulsarApi.proto#L259" target="_blank" rel="noopener noreferrer">18 or later</a>.</p></li><li><p>Configure the <a href="/docs/3.0.x/reference-configuration/"><code>brokerEntryMetadataInterceptors</code></a> parameter and set the <a href="/docs/3.0.x/reference-configuration/"><code>exposingBrokerEntryMetadataToClientEnabled</code></a> parameter to <code>true</code> in the <code>broker.conf</code> file.</p></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="message-metadata">Message metadata<a href="#message-metadata" class="hash-link" aria-label="Direct link to Message metadata" title="Direct link to Message metadata"></a></h2><p>Message metadata is stored alongside the application-specified payload as a serialized protobuf message. Metadata is created by the producer and passed without changes to the consumer.</p><table><thead><tr><th align="left">Field</th><th align="left">Required or optional</th><th align="left">Description</th></tr></thead><tbody><tr><td align="left"><code>producer_name</code></td><td align="left">Required</td><td align="left">The name of the producer that published the message</td></tr><tr><td align="left"><code>sequence_id</code></td><td align="left">Required</td><td align="left">The sequence ID of the message, assigned by producer</td></tr><tr><td align="left"><code>publish_time</code></td><td align="left">Required</td><td align="left">The publish timestamp in Unix time (i.e. as the number of milliseconds since January 1st, 1970 in UTC)</td></tr><tr><td align="left"><code>properties</code></td><td align="left">Required</td><td align="left">A sequence of key/value pairs (using the <a href="https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/proto/PulsarApi.proto#L32" target="_blank" rel="noopener noreferrer"><code>KeyValue</code></a> message). These are application-defined keys and values with no special meaning to Pulsar.</td></tr><tr><td align="left"><code>replicated_from</code></td><td align="left">Optional</td><td align="left">Indicates that the message has been replicated and specifies the name of the <a href="/docs/3.0.x/reference-terminology/#cluster">cluster</a> where the message was originally published</td></tr><tr><td align="left"><code>partition_key</code></td><td align="left">Optional</td><td align="left">While publishing on a partitioned topic, if the key is present, the hash of the key is used to determine which partition to choose. Partition key is used as the message key.</td></tr><tr><td align="left"><code>compression</code></td><td align="left">Optional</td><td align="left">Signals that payload has been compressed and with which compression library</td></tr><tr><td align="left"><code>uncompressed_size</code></td><td align="left">Optional</td><td align="left">If compression is used, the producer must fill the uncompressed size field with the original payload size</td></tr><tr><td align="left"><code>num_messages_in_batch</code></td><td align="left">Optional</td><td align="left">If this message is really a <a href="#batch-messages">batch</a> of multiple entries, this field must be set to the number of messages in the batch</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="batch-messages">Batch messages<a href="#batch-messages" class="hash-link" aria-label="Direct link to Batch messages" title="Direct link to Batch messages"></a></h3><p>When using batch messages, the payload will be containing a list of entries, |
| each of them with its individual metadata, defined by the <code>SingleMessageMetadata</code> |
| object.</p><p>For a single batch, the payload format will look like this:</p><table><thead><tr><th align="left">Field</th><th align="left">Required or optional</th><th align="left">Description</th></tr></thead><tbody><tr><td align="left"><code>metadataSizeN</code></td><td align="left">Required</td><td align="left">The size of the single message metadata serialized Protobuf</td></tr><tr><td align="left"><code>metadataN</code></td><td align="left">Required</td><td align="left">Single message metadata</td></tr><tr><td align="left"><code>payloadN</code></td><td align="left">Required</td><td align="left">Message payload passed by application</td></tr></tbody></table><p>Each metadata field looks like this;</p><table><thead><tr><th align="left">Field</th><th align="left">Required or optional</th><th align="left">Description</th></tr></thead><tbody><tr><td align="left"><code>properties</code></td><td align="left">Required</td><td align="left">Application-defined properties</td></tr><tr><td align="left"><code>partition key</code></td><td align="left">Optional</td><td align="left">Key to indicate the hashing to a particular partition</td></tr><tr><td align="left"><code>payload_size</code></td><td align="left">Required</td><td align="left">Size of the payload for the single message in the batch</td></tr></tbody></table><p>When compression is enabled, the whole batch will be compressed at once.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interactions">Interactions<a href="#interactions" class="hash-link" aria-label="Direct link to Interactions" title="Direct link to Interactions"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="connection-establishment">Connection establishment<a href="#connection-establishment" class="hash-link" aria-label="Direct link to Connection establishment" title="Direct link to Connection establishment"></a></h3><p>After opening a TCP connection to a broker, typically on port 6650, the client |
| is responsible to initiate the session.</p><p><img loading="lazy" alt="Connect interaction" src="/assets/images/binary-protocol-connect-62f0caa199b185efbab5182c49bdaf97.png" width="475" height="215" class="img_ev3q"></p><p>After receiving a <code>Connected</code> response from the broker, the client can |
| consider the connection ready to use. Alternatively, if the broker doesn't |
| validate the client authentication, it will reply with an <code>Error</code> command and |
| close the TCP connection.</p><p>Example:</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandConnect</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"client_version"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"Pulsar-Client-Java-v1.15.2"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"auth_method_name"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"my-authentication-plugin"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"auth_data"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"my-auth-data"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"protocol_version"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">6</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>client_version</code>: String-based identifier. Format is not enforced.</li><li><code>auth_method_name</code>: <em>(optional)</em> Name of the authentication plugin if auth is enabled.</li><li><code>auth_data</code>: <em>(optional)</em> Plugin specific authentication data.</li><li><code>protocol_version</code>: Indicates the protocol version supported by the client. Broker will not send commands introduced in newer revisions of the protocol. Broker might be enforcing a minimum version.</li><li><code>original_principal</code>: Added by the proxy. Regular clients are not expected to supply this value. When set and when authorization is enabled, the <code>auth_data</code> must map to one of the <code>proxyRoles</code> in the broker.conf.</li><li><code>original_auth_method</code>: Added by the proxy. Regular clients are not expected to supply this value.</li><li><code>original_auth_data</code>: Added by the proxy when configured to do so. Regular clients are not expected to supply this value.</li><li><code>proxy_version</code>: Added by the proxy. The proxy rejects <code>Connect</code> commands with this field present. Regular clients are not expected to supply this value. When authentication and authorization are enabled in the broker, only one of the configured <code>proxyRoles</code> has permission to supply this field. To be backward compatible, the broker does not require a <code>proxyRole</code> to supply this field.</li></ul><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandConnected</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"server_version"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"Pulsar-Broker-v1.15.2"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"protocol_version"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">6</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>server_version</code>: String identifier of broker version.</li><li><code>protocol_version</code>: Protocol version supported by the broker. Client |
| must not attempt to send commands introduced in newer revisions of the |
| protocol.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keep-alive">Keep Alive<a href="#keep-alive" class="hash-link" aria-label="Direct link to Keep Alive" title="Direct link to Keep Alive"></a></h3><p>To identify prolonged network partitions between clients and brokers or cases |
| in which a machine crashes without interrupting the TCP connection on the remote |
| end (eg: power outage, kernel panic, hard reboot...), we have introduced a |
| mechanism to probe for the availability status of the remote peer.</p><p>Both clients and brokers are sending <code>Ping</code> commands periodically and they will |
| close the socket if a <code>Pong</code> response is not received within a timeout (default |
| used by broker is 60s).</p><p>A valid implementation of a Pulsar client is not required to send the <code>Ping</code> |
| probe, though it is required to promptly reply after receiving one from the |
| broker in order to prevent the remote side from forcibly closing the TCP connection.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="producer">Producer<a href="#producer" class="hash-link" aria-label="Direct link to Producer" title="Direct link to Producer"></a></h3><p>In order to send messages, a client needs to establish a producer. When creating |
| a producer, the broker will first verify that this particular client is |
| authorized to publish on the topic.</p><p>Once the client gets confirmation of the producer creation, it can publish |
| messages to the broker, referring to the producer ID negotiated before.</p><p><img loading="lazy" alt="Producer interaction" src="/assets/images/binary-protocol-producer-2746d3914bebf293532476b046540132.png" width="469" height="416" class="img_ev3q"></p><p>If the client does not receive a response indicating producer creation success or failure, |
| the client should first send a command to close the original producer before sending a |
| command to re-attempt producer creation.</p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Before creating or connecting a producer, you need to perform <a href="#topic-lookup">topic lookup</a> first.</p></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-producer">Command Producer<a href="#command-producer" class="hash-link" aria-label="Direct link to Command Producer" title="Direct link to Command Producer"></a></h5><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandProducer</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"topic"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"persistent://my-property/my-cluster/my-namespace/my-topic"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"producer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>topic</code>: Complete topic name to where you want to create the producer on.</li><li><code>producer_id</code>: Client generated producer identifier. Needs to be unique |
| within the same connection.</li><li><code>request_id</code>: Identifier for this request. Used to match the response with |
| the originating request. Needs to be unique within the same connection.</li><li><code>producer_name</code>: <em>(optional)</em> If a producer name is specified, the name will |
| be used, otherwise the broker will generate a unique name. Generated |
| producer name is guaranteed to be globally unique. Implementations are |
| expected to let the broker generate a new producer name when the producer |
| is initially created, then reuse it when recreating the producer after |
| reconnections.</li></ul><p>The broker will reply with either <code>ProducerSuccess</code> or <code>Error</code> commands.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-producersuccess">Command ProducerSuccess<a href="#command-producersuccess" class="hash-link" aria-label="Direct link to Command ProducerSuccess" title="Direct link to Command ProducerSuccess"></a></h5><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandProducerSuccess</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"producer_name"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"generated-unique-producer-name"</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>request_id</code>: Original ID of the <code>CreateProducer</code> request.</li><li><code>producer_name</code>: Generated globally unique producer name or the name |
| specified by the client, if any.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-send">Command Send<a href="#command-send" class="hash-link" aria-label="Direct link to Command Send" title="Direct link to Command Send"></a></h5><p>Command <code>Send</code> is used to publish a new message within the context of an |
| already existing producer. If a producer has not yet been created for the |
| connection, the broker will terminate the connection. This command is used |
| in a frame that includes command as well as message payload, for which the |
| complete format is specified in the <a href="#message-commands">message commands</a> section.</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandSend</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"producer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"sequence_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">0</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"num_messages"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>producer_id</code>: The ID of an existing producer.</li><li><code>sequence_id</code>: Each message has an associated sequence ID which is expected |
| to be implemented with a counter starting at 0. The <code>SendReceipt</code> that |
| acknowledges the effective publishing of messages will refer to it by |
| its sequence ID.</li><li><code>num_messages</code>: <em>(optional)</em> Used when publishing a batch of messages at |
| once.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-sendreceipt">Command SendReceipt<a href="#command-sendreceipt" class="hash-link" aria-label="Direct link to Command SendReceipt" title="Direct link to Command SendReceipt"></a></h5><p>After a message has been persisted on the configured number of replicas, the |
| broker will send the acknowledgment receipt to the producer.</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandSendReceipt</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"producer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"sequence_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">0</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"message_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"ledgerId"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">123</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"entryId"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">456</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>producer_id</code>: The ID of producer originating the send request.</li><li><code>sequence_id</code>: The sequence ID of the published message.</li><li><code>message_id</code>: The message ID assigned by the system to the published message |
| Unique within a single cluster. Message ID is composed of 2 longs, <code>ledgerId</code> |
| and <code>entryId</code>, which reflects that this unique ID is assigned when appending |
| to a BookKeeper ledger.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-closeproducer">Command CloseProducer<a href="#command-closeproducer" class="hash-link" aria-label="Direct link to Command CloseProducer" title="Direct link to Command CloseProducer"></a></h5><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>This command can be sent by either producer or broker.</p></div></div><p>When receiving a <code>CloseProducer</code> command, the broker will stop accepting any |
| more messages for the producer, wait until all pending messages are persisted |
| and then reply <code>Success</code> to the client.</p><p>If the client does not receive a response to a <code>Producer</code> command within a timeout, |
| the client must first send a <code>CloseProducer</code> command before sending another |
| <code>Producer</code> command. The client does not need to await a response to the <code>CloseProducer</code> |
| command before sending the next <code>Producer</code> command.</p><p>The broker can send a <code>CloseProducer</code> command to client when it's performing |
| a graceful failover (eg: broker is being restarted, or the topic is being unloaded |
| by load balancer to be transferred to a different broker).</p><p>When receiving the <code>CloseProducer</code>, the client is expected to go through the |
| service discovery lookup again and recreate the producer again. The TCP |
| connection is not affected.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="consumer">Consumer<a href="#consumer" class="hash-link" aria-label="Direct link to Consumer" title="Direct link to Consumer"></a></h3><p>A consumer is used to attach to a subscription and consume messages from it. |
| After every reconnection, a client needs to subscribe to the topic. If a |
| subscription is not already there, a new one will be created.</p><p><img loading="lazy" alt="Consumer" src="/assets/images/binary-protocol-consumer-36dd73884fd5e453b4662e51ad844f1f.png" width="469" height="416" class="img_ev3q"></p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Before creating or connecting a consumer, you need to perform <a href="#topic-lookup">topic lookup</a> first.</p></div></div><p>If the client does not receive a response indicating consumer creation success or failure, |
| the client should first send a command to close the original consumer before sending a |
| command to re-attempt consumer creation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="flow-control">Flow control<a href="#flow-control" class="hash-link" aria-label="Direct link to Flow control" title="Direct link to Flow control"></a></h4><p>After the consumer is ready, the client needs to <em>give permission</em> to the |
| broker to push messages. This is done with the <code>Flow</code> command.</p><p>A <code>Flow</code> command gives additional <em>permits</em> to send messages to the consumer. |
| A typical consumer implementation will use a queue to accumulate these messages |
| before the application is ready to consume them.</p><p>After the application has dequeued half of the messages in the queue, the consumer |
| sends permits to the broker to ask for more messages (equals to half of the messages in the queue).</p><p>For example, if the queue size is 1000 and the consumer consumes 500 messages in the queue. |
| Then the consumer sends permits to the broker to ask for 500 messages.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-subscribe">Command Subscribe<a href="#command-subscribe" class="hash-link" aria-label="Direct link to Command Subscribe" title="Direct link to Command Subscribe"></a></h5><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandSubscribe</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"topic"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"persistent://my-property/my-cluster/my-namespace/my-topic"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"subscription"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"my-subscription-name"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"subType"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"Exclusive"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"consumer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>topic</code>: Complete topic name to where you want to create the consumer on.</li><li><code>subscription</code>: Subscription name.</li><li><code>subType</code>: Subscription type: Exclusive, Shared, Failover, Key_Shared.</li><li><code>consumer_id</code>: Client generated consumer identifier. Needs to be unique |
| within the same connection.</li><li><code>request_id</code>: Identifier for this request. Used to match the response with |
| the originating request. Needs to be unique within the same connection.</li><li><code>consumer_name</code>: <em>(optional)</em> Clients can specify a consumer name. This |
| name can be used to track a particular consumer in the stats. Also, in |
| Failover subscription type, the name is used to decide which consumer is |
| elected as <em>master</em> (the one receiving messages): consumers are sorted by |
| their consumer name and the first one is elected master.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-flow">Command Flow<a href="#command-flow" class="hash-link" aria-label="Direct link to Command Flow" title="Direct link to Command Flow"></a></h5><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandFlow</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"consumer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"messagePermits"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>consumer_id</code>: The ID of an already established consumer.</li><li><code>messagePermits</code>: The number of additional permits to grant to the broker for |
| pushing more messages.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-message">Command Message<a href="#command-message" class="hash-link" aria-label="Direct link to Command Message" title="Direct link to Command Message"></a></h5><p>Command <code>Message</code> is used by the broker to push messages to an existing consumer, |
| within the limits of the given permits.</p><p>This command is used in a frame that includes the message payload as well, for |
| which the complete format is specified in the <a href="#message-commands">message commands</a> |
| section.</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandMessage</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"consumer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"message_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"ledgerId"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">123</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"entryId"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">456</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-ack">Command Ack<a href="#command-ack" class="hash-link" aria-label="Direct link to Command Ack" title="Direct link to Command Ack"></a></h5><p>An <code>Ack</code> is used to signal to the broker that a given message has been |
| successfully processed by the application and can be discarded by the broker.</p><p>In addition, the broker will also maintain the consumer position based on the |
| acknowledged messages.</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandAck</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"consumer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"ack_type"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"Individual"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"message_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"ledgerId"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">123</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"entryId"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">456</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>consumer_id</code>: The ID of an already established consumer.</li><li><code>ack_type</code>: Acknowledgment type: <code>Individual</code> or <code>Cumulative</code>.</li><li><code>message_id</code>: The ID of the message to acknowledge.</li><li><code>validation_error</code>: <em>(optional)</em> Indicates that the consumer has discarded |
| the messages due to: <code>UncompressedSizeCorruption</code>, |
| <code>DecompressionError</code>, <code>ChecksumMismatch</code>, <code>BatchDeSerializeError</code>.</li><li><code>properties</code>: <em>(optional)</em> Reserved configuration items.</li><li><code>txnid_most_bits</code>: <em>(optional)</em> Same as Transaction Coordinator ID, <code>txnid_most_bits</code> and <code>txnid_least_bits</code> |
| uniquely identify a transaction.</li><li><code>txnid_least_bits</code>: <em>(optional)</em> The ID of the transaction opened in a transaction coordinator, |
| <code>txnid_most_bits</code> and <code>txnid_least_bits</code>uniquely identify a transaction.</li><li><code>request_id</code>: <em>(optional)</em> The ID for handling response and timeout.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-ackresponse">Command AckResponse<a href="#command-ackresponse" class="hash-link" aria-label="Direct link to Command AckResponse" title="Direct link to Command AckResponse"></a></h5><p>An <code>AckResponse</code> is the broker's response to acknowledge a request sent by the client. It contains the <code>consumer_id</code> sent in the request. |
| If a transaction is used, it contains both the Transaction ID and the Request ID that are sent in the request. |
| The client finishes the specific request according to the Request ID. |
| If the <code>error</code> field is set, it indicates that the request has failed.</p><p>An example of <code>AckResponse</code> with redirection:</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandAckResponse</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"consumer_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"txnid_least_bits"</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number">0</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"txnid_most_bits"</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-closeconsumer">Command CloseConsumer<a href="#command-closeconsumer" class="hash-link" aria-label="Direct link to Command CloseConsumer" title="Direct link to Command CloseConsumer"></a></h5><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>This command can be sent by either producer or broker.</p></div></div><p>This command behaves the same as <a href="#command-closeproducer"><code>CloseProducer</code></a></p><p>If the client does not receive a response to a <code>Subscribe</code> command within a timeout, |
| the client must first send a <code>CloseConsumer</code> command before sending another |
| <code>Subscribe</code> command. The client does not need to await a response to the <code>CloseConsumer</code> |
| command before sending the next <code>Subscribe</code> command.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-redeliverunacknowledgedmessages">Command RedeliverUnacknowledgedMessages<a href="#command-redeliverunacknowledgedmessages" class="hash-link" aria-label="Direct link to Command RedeliverUnacknowledgedMessages" title="Direct link to Command RedeliverUnacknowledgedMessages"></a></h5><p>A consumer can ask the broker to redeliver some or all of the pending messages |
| that were pushed to that particular consumer and not yet acknowledged.</p><p>The protobuf object accepts a list of message IDs that the consumer wants to |
| be redelivered. If the list is empty, the broker will redeliver all the |
| pending messages.</p><p>On redelivery, messages can be sent to the same consumer or, in the case of a |
| shared subscription, spread across all available consumers.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-reachedendoftopic">Command ReachedEndOfTopic<a href="#command-reachedendoftopic" class="hash-link" aria-label="Direct link to Command ReachedEndOfTopic" title="Direct link to Command ReachedEndOfTopic"></a></h5><p>This is sent by a broker to a particular consumer, whenever the topic |
| has been "terminated" and all the messages on the subscription were |
| acknowledged.</p><p>The client should use this command to notify the application that no more |
| messages are coming from the consumer.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-consumerstats">Command ConsumerStats<a href="#command-consumerstats" class="hash-link" aria-label="Direct link to Command ConsumerStats" title="Direct link to Command ConsumerStats"></a></h5><p>This command is sent by the client to retrieve Subscriber and Consumer level |
| stats from the broker.</p><p>Fields:</p><ul><li><code>request_id</code>: The ID of the request, used to correlate the request |
| and the response.</li><li><code>consumer_id</code>: The ID of an already established consumer.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-consumerstatsresponse">Command ConsumerStatsResponse<a href="#command-consumerstatsresponse" class="hash-link" aria-label="Direct link to Command ConsumerStatsResponse" title="Direct link to Command ConsumerStatsResponse"></a></h5><p>This is the broker's response to ConsumerStats request by the client. |
| It contains the Subscriber and Consumer level stats of the <code>consumer_id</code> sent in the request. |
| If the <code>error_code</code> or the <code>error_message</code> field is set it indicates that the request has failed.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-unsubscribe">Command Unsubscribe<a href="#command-unsubscribe" class="hash-link" aria-label="Direct link to Command Unsubscribe" title="Direct link to Command Unsubscribe"></a></h5><p>This command is sent by the client to unsubscribe the <code>consumer_id</code> from the associated topic.</p><p>Fields:</p><ul><li><code>request_id</code>: The ID of the request.</li><li><code>consumer_id</code>: The ID of an already established consumer that needs to unsubscribe.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="service-discovery">Service discovery<a href="#service-discovery" class="hash-link" aria-label="Direct link to Service discovery" title="Direct link to Service discovery"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="topic-lookup">Topic lookup<a href="#topic-lookup" class="hash-link" aria-label="Direct link to Topic lookup" title="Direct link to Topic lookup"></a></h3><p>Topic lookup needs to be performed each time a client needs to create or |
| reconnect a producer or a consumer. Lookup is used to discover which particular |
| broker is serving the topic we are about to use.</p><p>Lookup can be done with a REST call as described in the <a href="/docs/3.0.x/admin-api-topics/#look-up-topics-owner-broker">admin API</a> docs.</p><p>Since Pulsar-1.16 it is also possible to perform the lookup within the binary protocol.</p><p>For the sake of example, let's assume we have a service discovery component |
| running at <code>pulsar://broker.example.com:6650</code></p><p>Individual brokers will be running at <code>pulsar://broker-1.example.com:6650</code>, |
| <code>pulsar://broker-2.example.com:6650</code>, ...</p><p>A client can use a connection to the discovery service host to issue a |
| <code>LookupTopic</code> command. The response can either be a broker hostname to |
| connect to, or a broker hostname to which retry the lookup.</p><p>The <code>LookupTopic</code> command has to be used in a connection that has already |
| gone through the <code>Connect</code> / <code>Connected</code> initial handshake.</p><p><img loading="lazy" alt="Topic lookup" src="/assets/images/binary-protocol-topic-lookup-f013216a8dae04823eb9d39a0f2e264e.png" width="587" height="307" class="img_ev3q"></p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandLookupTopic</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"topic"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"persistent://my-property/my-cluster/my-namespace/my-topic"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"authoritative"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token boolean">false</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>topic</code>: The topic name to lookup.</li><li><code>request_id</code>: The ID of the request that will be passed with its response.</li><li><code>authoritative</code>: Initial lookup request should use false. When following a |
| redirect response, client should pass the same value contained in the |
| response.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="lookuptopicresponse">LookupTopicResponse<a href="#lookuptopicresponse" class="hash-link" aria-label="Direct link to LookupTopicResponse" title="Direct link to LookupTopicResponse"></a></h5><p>An example of response with successful lookup:</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandLookupTopicResponse</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"response"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"Connect"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"brokerServiceUrl"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"pulsar://broker-1.example.com:6650"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"brokerServiceUrlTls"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"pulsar+ssl://broker-1.example.com:6651"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"authoritative"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This is an example of lookup response with redirection:</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandLookupTopicResponse</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"response"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"Redirect"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"brokerServiceUrl"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"pulsar://broker-2.example.com:6650"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"brokerServiceUrlTls"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"pulsar+ssl://broker-2.example.com:6651"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"authoritative"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In this second case, we need to reissue the <code>LookupTopic</code> command request |
| to <code>broker-2.example.com</code> and this broker will be able to give a definitive |
| answer to the lookup request.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="partitioned-topics-discovery">Partitioned topics discovery<a href="#partitioned-topics-discovery" class="hash-link" aria-label="Direct link to Partitioned topics discovery" title="Direct link to Partitioned topics discovery"></a></h3><p>Partitioned topics metadata discovery is used to find out if a topic is a |
| "partitioned topic" and how many partitions were set up.</p><p>If the topic is marked as "partitioned", the client is expected to create |
| multiple producers or consumers, one for each partition, using the <code>partition-X</code> |
| suffix.</p><p>This information only needs to be retrieved the first time a producer or |
| consumer is created. There is no need to do this after reconnections.</p><p>The discovery of partitioned topics metadata works very similarly to the topic |
| lookup. The client send a request to the service discovery address and the |
| response will contain actual metadata.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-partitionedtopicmetadata">Command PartitionedTopicMetadata<a href="#command-partitionedtopicmetadata" class="hash-link" aria-label="Direct link to Command PartitionedTopicMetadata" title="Direct link to Command PartitionedTopicMetadata"></a></h5><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandPartitionedTopicMetadata</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"topic"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"persistent://my-property/my-cluster/my-namespace/my-topic"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Fields:</p><ul><li><code>topic</code>: The topic for which to check the partitions metadata.</li><li><code>request_id</code>: The ID of the request that will be passed with its response.</li></ul><h5 class="anchor anchorWithStickyNavbar_LWe7" id="command-partitionedtopicmetadataresponse">Command PartitionedTopicMetadataResponse<a href="#command-partitionedtopicmetadataresponse" class="hash-link" aria-label="Direct link to Command PartitionedTopicMetadataResponse" title="Direct link to Command PartitionedTopicMetadataResponse"></a></h5><p>An example of response with metadata:</p><div class="language-protobuf codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-protobuf codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">message</span><span class="token plain"> </span><span class="token class-name">CommandPartitionedTopicMetadataResponse</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"request_id"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">1</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"response"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"Success"</span><span class="token punctuation" style="color:rgb(248, 248, 242)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><span class="token string" style="color:rgb(255, 121, 198)">"partitions"</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(248, 248, 242)">:</span><span class="token plain"> </span><span class="token number">32</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(248, 248, 242)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="protobuf-interface">Protobuf interface<a href="#protobuf-interface" class="hash-link" aria-label="Direct link to Protobuf interface" title="Direct link to Protobuf interface"></a></h2><p>All Pulsar's Protobuf definitions can be found <a href="https://github.com/apache/pulsar/tree/master//pulsar-common/src/main/proto/PulsarApi.proto" target="_blank" rel="noopener noreferrer">here</a>.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/pulsar-site/edit/main/versioned_docs/version-3.0.x/developing-binary-protocol.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_vwxv"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/3.0.x/develop-tools/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Simulation tools</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/3.0.x/develop-load-manager/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Modular load manager</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_jeP5 thin-scrollbar theme-doc-toc-desktop"><div class="border"><div style="color:var(--ifm-toc-link-color)">Was this helpful?</div><div style="border-width:1px;padding:3px;display:flex"><div style="justify-content:center;display:flex;border-radius:99999px;width:2.5rem;height:2.5rem;cursor:pointer;background:;color:"><svg style="width:initial;height:initial" width="12" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.086 1.594A1 1 0 0 1 11 1a4 4 0 0 1 4 4v3h4.655a3 3 0 0 1 2.994 3.45l-1.38 9A3.002 3.002 0 0 1 18.275 23H4a3 3 0 0 1-3-3v-7a3 3 0 0 1 3-3h2.35l3.736-8.406ZM8 11.212l3.608-8.117A2 2 0 0 1 13 5v4a1 1 0 0 0 1 1h5.671a1 1 0 0 1 1 1.15l-1.38 9a1 1 0 0 1-1 .85H8v-9.788ZM6 21v-9H4a1 1 0 0 0-1 1v7a1 1 0 0 0 1 1h2Z" fill="currentColor"></path></svg></div><div style="justify-content:center;display:flex;border-radius:99999px;width:2.5rem;height:2.5rem;cursor:pointer;background:;color:"><svg style="width:initial;height:initial" width="12" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M20.563 3.316A1.31 1.31 0 0 0 19.687 3h-1.688v9h1.688a1.31 1.31 0 0 0 1.312-1.077V4.077a1.31 1.31 0 0 0-.436-.761ZM16 12.788l-3.608 8.117A1.999 1.999 0 0 1 11 19v-4a1 1 0 0 0-1-1H4.328a1.002 1.002 0 0 1-1-1.15l1.38-9a1 1 0 0 1 1-.85h10.291v9.788ZM19.661 1a3.31 3.31 0 0 1 3.329 2.866c.006.044.01.09.01.134v7c0 .045-.004.09-.01.134A3.31 3.31 0 0 1 19.661 14h-2.012l-3.736 8.406a1 1 0 0 1-.914.594 4 4 0 0 1-4-4v-3H4.344a3 3 0 0 1-2.994-3.45l1.38-9A3.002 3.002 0 0 1 5.724 1h13.937Z" fill="currentColor"></path></svg></div></div><div class="Actions_uugI"><a target="_blank" class="Action_iBHd" href="https://github.com/apache/pulsar/issues/new?assignees=&labels=doc-required&projects=&template=doc.yml&title=%5BDoc%5D+">💡 Suggest changes</a><a target="_blank" class="Action_iBHd" href="https://github.com/apache/pulsar/discussions/new?category=q-a">🛟 Get support</a></div></div><ul class="table-of-contents table-of-contents__left-border"><li><a href="#framing" class="table-of-contents__link toc-highlight">Framing</a><ul><li><a href="#simple-commands" class="table-of-contents__link toc-highlight">Simple commands</a></li><li><a href="#message-commands" class="table-of-contents__link toc-highlight">Message commands</a></li></ul></li><li><a href="#broker-entry-metadata" class="table-of-contents__link toc-highlight">Broker entry metadata</a></li><li><a href="#message-metadata" class="table-of-contents__link toc-highlight">Message metadata</a><ul><li><a href="#batch-messages" class="table-of-contents__link toc-highlight">Batch messages</a></li></ul></li><li><a href="#interactions" class="table-of-contents__link toc-highlight">Interactions</a><ul><li><a href="#connection-establishment" class="table-of-contents__link toc-highlight">Connection establishment</a></li><li><a href="#keep-alive" class="table-of-contents__link toc-highlight">Keep Alive</a></li><li><a href="#producer" class="table-of-contents__link toc-highlight">Producer</a></li><li><a href="#consumer" class="table-of-contents__link toc-highlight">Consumer</a></li></ul></li><li><a href="#service-discovery" class="table-of-contents__link toc-highlight">Service discovery</a><ul><li><a href="#topic-lookup" class="table-of-contents__link toc-highlight">Topic lookup</a></li><li><a href="#partitioned-topics-discovery" class="table-of-contents__link toc-highlight">Partitioned topics discovery</a></li></ul></li><li><a href="#protobuf-interface" class="table-of-contents__link toc-highlight">Protobuf interface</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Foundation<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/events/current-event.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Events<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/security/">Security</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Privacy<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/contact/">Contact</a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"> |
| <div class="social-icons"> |
| <a target="_blank" href="https://communityinviter.com/apps/apache-pulsar/apache-pulsar" aria-label="Join the Apache Pulsar Slack workspace"> |
| <img alt="Slack logo" src="/img/slack-white.svg" width="26"> |
| </a> |
| <a target="_blank" href="https://github.com/apache/pulsar/" aria-label="View the Apache Pulsar project on GitHub"> |
| <img alt="GitHub logo" src="/img/github-white.svg" width="26"> |
| </a> |
| </div> |
| </li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a class="footerLogoLink_BH7S" href="/"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></a></div><div class="footer__copyright"> |
| <div> |
| <img class="footer-apache-logo" src="/img/feather-logo-white.svg" alt="" width="20"> |
| The Apache Software Foundation |
| </div> |
| <p>Apache Pulsar is available under the Apache License, version 2.0. Apache Pulsar is an open-source, distributed messaging and streaming platform built for the cloud.</p> |
| <p>Copyright © 2024 The Apache Software Foundation. All Rights Reserved. Apache, Pulsar, Apache Pulsar, and the Apache feather logo are trademarks or registered trademarks of The Apache Software Foundation.</p> |
| </div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.f1ab6cb0.js"></script> |
| <script src="/assets/js/main.9801eec2.js"></script> |
| </body> |
| </html> |
