| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-3.2.x plugin-docs plugin-id-default docs-doc-id-txn-how"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.0"> |
| <title data-rh="true">How transactions work? | Apache Pulsar</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" name="twitter:image" content="https://pulsar.apache.org/img/pulsar-social-media-card.png"><meta data-rh="true" property="og:url" content="https://pulsar.apache.org/docs/3.2.x/txn-how/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="3.2.x"><meta data-rh="true" name="docusaurus_tag" content="docs-default-3.2.x"><meta data-rh="true" name="docsearch:version" content="3.2.x"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-3.2.x"><meta data-rh="true" property="og:title" content="How transactions work? | Apache Pulsar"><meta data-rh="true" name="description" content="Learn the working principles of transactions in Pulsar."><meta data-rh="true" property="og:description" content="Learn the working principles of transactions in Pulsar."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/docs/3.2.x/txn-how/" hreflang="en"><link data-rh="true" rel="alternate" href="https://pulsar.apache.org/docs/3.2.x/txn-how/" hreflang="x-default"><link data-rh="true" rel="canonical" href="https://pulsar.apache.org/docs/txn-how/"><link data-rh="true" rel="preconnect" href="https://WK2YL0SALL-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Pulsar RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Pulsar Atom Feed"> |
| |
| |
| |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache Pulsar" href="/opensearch.xml"> |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="/css/katex-0.13.24.min.css" media="print" onload="this.media='all'"> |
| <script src="/js/sine-waves.min.js" async></script> |
| <script src="/js/matomo-agent.js"></script><link rel="stylesheet" href="/assets/css/styles.b0f65ef3.css"> |
| <link rel="preload" href="/assets/js/runtime~main.1d0ed2a7.js" as="script"> |
| <link rel="preload" href="/assets/js/main.e07a0c68.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:#282826;color:#fff" role="banner"><div class="content_knG7 announcementBarContent_xLdY"> |
| <a class="announcement-bar" href="https://registration.socio.events/e/pulsarvirtualsummiteurope2024" target="_blank"> |
| <div class="announcement-bar__content"> |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 33 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <path d="M6.5 19.6001H16.1L15.3 29.2001L26.5 12.4H17.06L18.1 2.80005L6.5 19.6001Z" stroke="#F7F7F7" stroke-width="1.5" stroke-linejoin="round"/> |
| </svg> |
| |
| </svg> |
| |
| <span> |
| Get your free pass for Pulsar Virtual Summit Europe 2024 on May 14, 2024 🗓️ |
| </span> |
| |
| <svg class="announcement-bar__icon"> |
| |
| <svg viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg"> |
| <rect width="20" height="20" transform="translate(6 6)" fill="white" fill-opacity="0.01"/> |
| <path d="M17.6667 10.1667L23.5 16.0001M23.5 16.0001L17.6667 21.8334M23.5 16.0001L8.5 16.0001" stroke="white" stroke-linecap="round" stroke-linejoin="round"/> |
| <rect x="0.5" y="0.5" width="31" height="31" rx="15.5" stroke="white"/> |
| </svg> |
| |
| </svg> |
| </div> |
| </a> |
| </div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--light_HNdA" height="25" width="127"><img src="/img/logo-black.svg" alt="Apache Pulsar logo" class="themedImage_ToTc themedImage--dark_i4oU" height="25" width="127"></div><b class="navbar__title text--truncate"></b></a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Get Started</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/3.2.x/concepts-overview/">Concepts</a></li><li><a class="dropdown__link" href="/docs/3.2.x/">Quickstart</a></li><li><a class="dropdown__link" href="/ecosystem/">Ecosystem</a></li></ul></div><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/docs/3.2.x/">Docs</a><a class="navbar__item navbar__link" href="/features/">Features</a><a class="navbar__item navbar__link" href="/use-cases/">Use Cases</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link community-dropdown">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link scroll-link scroll-welcome" id="scroll-welcome" href="/community/">Welcome</a></li><li><a class="dropdown__link scroll-link scroll-discussions" id="scroll-discussions" href="/community/#section-discussions">Discussions</a></li><li><a class="dropdown__link scroll-link" id="scroll-governance" href="/community/#section-governance">Governance</a></li><li><a class="dropdown__link scroll-link" id="scroll-community" href="/community/#section-community">Meet the Community</a></li><li><a class="dropdown__link scroll-link" id="scroll-contribute" href="/community/#section-contribute">Contribute</a></li><li><a class="dropdown__link" href="/contribute/">Contribution Guide</a></li><li><a href="https://github.com/apache/pulsar/wiki" target="_blank" rel="noopener noreferrer" class="dropdown__link">Wiki</a></li><li><a href="https://github.com/apache/pulsar/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Issue Tracking</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Learn</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/blog/">Blog</a></li><li><a class="dropdown__link" href="/books/">Books</a></li><li><a class="dropdown__link" href="/case-studies/">Case Studies</a></li><li><a class="dropdown__link" href="/articles/">Articles</a></li><li><a class="dropdown__link" href="/presentations/">Presentations</a></li><li><a class="dropdown__link" href="/events/">Events</a></li></ul></div></div><div class="navbar__items navbar__items--right"><a class="navbar__item navbar__link navbar_download_button" href="/download/">Download</a><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20" aria-hidden="true"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_mhZE"><div class="sidebarVersionSwitch_EHpo">Version:<div class="navbar__item dropdown dropdown--hoverable"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/3.2.x/">3.2.x</a><ul class="dropdown__menu"><li><a class="dropdown__link">Next</a></li><li><a class="dropdown__link">3.2.x</a></li><li><a class="dropdown__link">3.0.x LTS</a></li><li><a class="dropdown__link">Others</a></li></ul></div></div><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/3.2.x/">About</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.2.x/getting-started-home/">Get Started</a><button aria-label="Toggle the collapsible sidebar category 'Get Started'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/concepts-overview/">Concepts and Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/schema-overview/">Pulsar Schema</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/functions-overview/">Pulsar Functions</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/io-overview/">Pulsar IO</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/sql-overview/">Pulsar SQL</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/tiered-storage-overview/">Tiered Storage</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/3.2.x/txn-why/">Transactions</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.2.x/txn-why/">Why transactions?</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.2.x/txn-what/">Concept</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.2.x/txn-use/">Get started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.2.x/txn-advanced-features/">Advanced features</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/3.2.x/txn-monitor/">Monitoring</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/3.2.x/txn-how/">Working principles</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.2.x/install-deploy-upgrade-landing/">Deployment</a><button aria-label="Toggle the collapsible sidebar category 'Deployment'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/administration-zk-bk/">Administration</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/administration-stats/">Observability</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/security-overview/">Security</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/performance-pulsar-perf/">Performance</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/client-libraries/">Client Libraries</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/admin-api-overview/">Admin API</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/3.2.x/adaptors-kafka/">Adaptors</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.2.x/how-to-landing/">Tutorials</a><button aria-label="Toggle the collapsible sidebar category 'Tutorials'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.2.x/developers-landing/">Development</a><button aria-label="Toggle the collapsible sidebar category 'Development'" type="button" class="clean-btn menu__caret"></button></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist" aria-expanded="false" href="/docs/3.2.x/reference-landing/">Reference</a><button aria-label="Toggle the collapsible sidebar category 'Reference'" type="button" class="clean-btn menu__caret"></button></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Transactions</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Working principles</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 3.2.x</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>How transactions work?</h1></header><p>This section describes transaction components and how the components work together. For the complete design details, see <a href="https://docs.google.com/document/d/145VYp09JKTw9jAT-7yNyFU255FptB2_B2Fye100ZXDI/edit#heading=h.bm5ainqxosrx" target="_blank" rel="noopener noreferrer">PIP-31: Transactional Streaming</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="key-concept">Key concept<a href="#key-concept" class="hash-link" aria-label="Direct link to Key concept" title="Direct link to Key concept"></a></h2><p>It is important to know the following key concepts, which is a prerequisite for understanding how transactions work.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transaction-coordinator">Transaction coordinator<a href="#transaction-coordinator" class="hash-link" aria-label="Direct link to Transaction coordinator" title="Direct link to Transaction coordinator"></a></h3><p>The transaction coordinator (TC) is a module running inside a Pulsar broker.</p><ul><li><p>It maintains the entire life cycle of transactions and prevents a transaction from getting into an incorrect status.</p></li><li><p>It handles transaction timeout, and ensures that the transaction is aborted after a transaction timeout.</p></li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transaction-log">Transaction log<a href="#transaction-log" class="hash-link" aria-label="Direct link to Transaction log" title="Direct link to Transaction log"></a></h3><p>All the transaction metadata persists in the transaction log. The transaction log is backed by a Pulsar topic. If the transaction coordinator crashes, it can restore the transaction metadata from the transaction log.</p><p>The transaction log stores the transaction status rather than actual messages in the transaction (the actual messages are stored in the actual topic partitions).</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transaction-buffer">Transaction buffer<a href="#transaction-buffer" class="hash-link" aria-label="Direct link to Transaction buffer" title="Direct link to Transaction buffer"></a></h3><p>Messages produced to a topic partition within a transaction are stored in the transaction buffer (TB) of that topic partition. The messages in the transaction buffer are not visible to consumers until the transactions are committed. The messages in the transaction buffer are discarded when the transactions are aborted.</p><p>Transaction buffer stores all ongoing and aborted transactions in memory. All messages are sent to the actual partitioned Pulsar topics. After transactions are committed, the messages in the transaction buffer are materialized (visible) to consumers. When the transactions are aborted, the messages in the transaction buffer are discarded.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transaction-id">Transaction ID<a href="#transaction-id" class="hash-link" aria-label="Direct link to Transaction ID" title="Direct link to Transaction ID"></a></h3><p>Transaction ID (TxnID) identifies a unique transaction in Pulsar. The transaction ID is 128-bit. The highest 16 bits are reserved for the ID of the transaction coordinator, and the remaining bits are used for monotonically increasing numbers in each transaction coordinator. It is easy to locate the transaction crash with the TxnID.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pending-acknowledge-state">Pending acknowledge state<a href="#pending-acknowledge-state" class="hash-link" aria-label="Direct link to Pending acknowledge state" title="Direct link to Pending acknowledge state"></a></h3><p>Pending acknowledge state maintains message acknowledgments within a transaction before a transaction completes. If a message is in the pending acknowledge state, the message cannot be acknowledged by other transactions until the message is removed from the pending acknowledge state.</p><p>The pending acknowledge state is persisted in the pending acknowledge log (cursor ledger). A new broker can restore the state from the pending acknowledge log to ensure the acknowledgment is not lost.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-flow">Data flow<a href="#data-flow" class="hash-link" aria-label="Direct link to Data flow" title="Direct link to Data flow"></a></h2><p>To help you debug or tune the transaction for better performance, review the following diagrams and descriptions.</p><p>The data flow of a transaction can be split into several steps.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-begin-a-transaction">1. Begin a transaction<a href="#1-begin-a-transaction" class="hash-link" aria-label="Direct link to 1. Begin a transaction" title="Direct link to 1. Begin a transaction"></a></h3><p>Before introducing the transaction in Pulsar, a producer is created and then messages are sent to brokers and stored in data logs.</p><p><img loading="lazy" src="/assets/images/txn-3-751a2bc51f91299f6c546b647c2f632c.png" width="1564" height="856" class="img_ev3q"></p><p>Let's walk through the steps for <em>beginning a transaction</em>.</p><table><thead><tr><th>Step</th><th>Description</th></tr></thead><tbody><tr><td>1.1</td><td>The first step is that the Pulsar client finds the transaction coordinator.</td></tr><tr><td>1.2</td><td>The transaction coordinator allocates a transaction ID for the transaction. In the transaction log, the transaction is logged with its transaction ID and status (OPEN), which ensures the transaction status is persisted regardless of transaction coordinator crashes.</td></tr><tr><td>1.3</td><td>The transaction log sends the result of persisting the transaction ID to the transaction coordinator.</td></tr><tr><td>1.4</td><td>After the transaction status entry is logged, the transaction coordinator brings the transaction ID back to the Pulsar client.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-publish-messages-with-a-transaction">2. Publish messages with a transaction<a href="#2-publish-messages-with-a-transaction" class="hash-link" aria-label="Direct link to 2. Publish messages with a transaction" title="Direct link to 2. Publish messages with a transaction"></a></h3><p>In this stage, the Pulsar client enters a transaction loop, repeating the <code>consume-process-produce</code> operation for all the messages that comprise the transaction. This is a long phase and is potentially composed of multiple produce and acknowledgment requests.</p><p><img loading="lazy" alt="Workflow of publishing messages with a transaction in Pulsar" src="/assets/images/txn-4-f7adc6fb4ff184199a981fc32dd2311e.png" width="1338" height="694" class="img_ev3q"></p><p>Let's walk through the steps for <em>publishing messages with a transaction</em>.</p><table><thead><tr><th>Step</th><th>Description</th></tr></thead><tbody><tr><td>2.1.1</td><td>Before the Pulsar client produces messages to a new topic partition, it sends a request to the transaction coordinator to add the partition to the transaction.</td></tr><tr><td>2.1.2</td><td>The transaction coordinator logs the partition changes of the transaction into the transaction log for durability, which ensures the transaction coordinator knows all the partitions that a transaction is handling. The transaction coordinator can commit or abort changes on each partition at the end-partition phase.</td></tr><tr><td>2.1.3</td><td>The transaction log sends the result of logging the new partition (used for producing messages) to the transaction coordinator.</td></tr><tr><td>2.1.4</td><td>The transaction coordinator sends the result of adding a new produced partition to the transaction.</td></tr><tr><td>2.2.1</td><td>The Pulsar client starts producing messages to partitions. The flow of this part is the same as the normal flow of producing messages except that the batch of messages produced by a transaction contains transaction IDs.</td></tr><tr><td>2.2.2</td><td>The broker writes messages to a partition.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-acknowledge-messages-with-a-transaction">3. Acknowledge messages with a transaction<a href="#3-acknowledge-messages-with-a-transaction" class="hash-link" aria-label="Direct link to 3. Acknowledge messages with a transaction" title="Direct link to 3. Acknowledge messages with a transaction"></a></h3><p>In this phase, the Pulsar client sends a request to the transaction coordinator and a new subscription is acknowledged as a part of a transaction.</p><p><img loading="lazy" alt="Workflow of acknowledging messages with a transaction in Pulsar" src="/assets/images/txn-5-66e33b5b6ba3d900a1635cb268a38b35.png" width="1292" height="694" class="img_ev3q"></p><p>Let's walk through the steps for <em>acknowledging messages with a transaction</em>.</p><table><thead><tr><th>Step</th><th>Description</th></tr></thead><tbody><tr><td>3.1.1</td><td>The Pulsar client sends a request to add an acknowledged subscription to the transaction coordinator.</td></tr><tr><td>3.1.2</td><td>The transaction coordinator logs the addition of subscription, which ensures that it knows all subscriptions handled by a transaction and can commit or abort changes on each subscription at the end phase.</td></tr><tr><td>3.1.3</td><td>The transaction log sends the result of logging the new partition (used for acknowledging messages) to the transaction coordinator.</td></tr><tr><td>3.1.4</td><td>The transaction coordinator sends the result of adding the new acknowledged partition to the transaction.</td></tr><tr><td>3.2</td><td>The Pulsar client acknowledges messages on the subscription. The flow of this part is the same as the normal flow of acknowledging messages except that the acknowledged request carries a transaction ID.</td></tr><tr><td>3.3</td><td>The broker receiving the acknowledgment request checks if the acknowledgment belongs to a transaction or not.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-end-a-transaction">4. End a transaction<a href="#4-end-a-transaction" class="hash-link" aria-label="Direct link to 4. End a transaction" title="Direct link to 4. End a transaction"></a></h3><p>At the end of a transaction, the Pulsar client decides to commit or abort the transaction. The transaction can be aborted when a conflict is detected in acknowledging messages.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="41-end-transaction-request">4.1 End transaction request<a href="#41-end-transaction-request" class="hash-link" aria-label="Direct link to 4.1 End transaction request" title="Direct link to 4.1 End transaction request"></a></h4><p>When the Pulsar client finishes a transaction, it issues an end transaction request.</p><p><img loading="lazy" alt="Workflow of ending a transaction request in Pulsar" src="/assets/images/txn-6-ac44126d5410be548e44717d2cc056fa.png" width="1284" height="694" class="img_ev3q"></p><p>Let's walk through the steps for <em>ending the transaction</em>.</p><table><thead><tr><th>Step</th><th>Description</th></tr></thead><tbody><tr><td>4.1.1</td><td>The Pulsar client issues an end transaction request (with a field indicating whether the transaction is to be committed or aborted) to the transaction coordinator.</td></tr><tr><td>4.1.2</td><td>The transaction coordinator writes a COMMITTING or ABORTING message to its transaction log.</td></tr><tr><td>4.1.3</td><td>The transaction log sends the result of logging the committing or aborting status.</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="42-finalize-a-transaction">4.2 Finalize a transaction<a href="#42-finalize-a-transaction" class="hash-link" aria-label="Direct link to 4.2 Finalize a transaction" title="Direct link to 4.2 Finalize a transaction"></a></h4><p>The transaction coordinator starts the process of committing or aborting messages to all the partitions involved in this transaction.</p><p><img loading="lazy" alt="Workflow of finalizing a transaction in Pulsar" src="/assets/images/txn-7-229fdd1904b8c411e77d48fe1c3fee65.png" width="1354" height="690" class="img_ev3q"></p><p>Let's walk through the steps for <em>finalizing a transaction</em>.</p><table><thead><tr><th>Step</th><th>Description</th></tr></thead><tbody><tr><td>4.2.1</td><td>The transaction coordinator commits transactions on subscriptions and commits transactions on partitions at the same time.</td></tr><tr><td>4.2.2</td><td>The broker (produce) writes produced committed markers to the actual partitions. At the same time, the broker (ack) writes acked committed marks to the subscription pending ack partitions.</td></tr><tr><td>4.2.3</td><td>The data log sends the result of writing produced committed marks to the broker. At the same time, pending ack data log sends the result of writing acked committed marks to the broker. The cursor moves to the next position.</td></tr></tbody></table><h4 class="anchor anchorWithStickyNavbar_LWe7" id="43-mark-a-transaction-as-committed-or-aborted">4.3 Mark a transaction as COMMITTED or ABORTED<a href="#43-mark-a-transaction-as-committed-or-aborted" class="hash-link" aria-label="Direct link to 4.3 Mark a transaction as COMMITTED or ABORTED" title="Direct link to 4.3 Mark a transaction as COMMITTED or ABORTED"></a></h4><p>The transaction coordinator writes the final transaction status to the transaction log to complete the transaction.</p><p><img loading="lazy" alt="Workflow of marking a transaction request in Pulsar" src="/assets/images/txn-8-d49405f853142c9762c4caaa8f862b4e.png" width="1262" height="692" class="img_ev3q"></p><p>Let's walk through the steps for <em>marking a transaction as COMMITTED or ABORTED</em>.</p><table><thead><tr><th>Step</th><th>Description</th></tr></thead><tbody><tr><td>4.3.1</td><td>After all produced messages and acknowledgments to all partitions involved in this transaction have been successfully committed or aborted, the transaction coordinator writes the final COMMITTED or ABORTED transaction status messages to its transaction log, indicating that the transaction is complete. All the messages associated with the transaction in its transaction log can be safely removed.</td></tr><tr><td>4.3.2</td><td>The transaction log sends the result of the committed transaction to the transaction coordinator.</td></tr><tr><td>4.3.3</td><td>The transaction coordinator sends the result of the committed transaction to the Pulsar client.</td></tr></tbody></table></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/pulsar-site/edit/main/versioned_docs/version-3.2.x/txn-how.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_vwxv"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/3.2.x/txn-monitor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Monitoring</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/3.2.x/install-deploy-upgrade-landing/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tutorials</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_jeP5 thin-scrollbar theme-doc-toc-desktop"><div class="border"><div style="color:var(--ifm-toc-link-color)">Was this helpful?</div><div style="border-width:1px;padding:3px;display:flex"><div style="justify-content:center;display:flex;border-radius:99999px;width:2.5rem;height:2.5rem;cursor:pointer;background:;color:"><svg style="width:initial;height:initial" width="12" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.086 1.594A1 1 0 0 1 11 1a4 4 0 0 1 4 4v3h4.655a3 3 0 0 1 2.994 3.45l-1.38 9A3.002 3.002 0 0 1 18.275 23H4a3 3 0 0 1-3-3v-7a3 3 0 0 1 3-3h2.35l3.736-8.406ZM8 11.212l3.608-8.117A2 2 0 0 1 13 5v4a1 1 0 0 0 1 1h5.671a1 1 0 0 1 1 1.15l-1.38 9a1 1 0 0 1-1 .85H8v-9.788ZM6 21v-9H4a1 1 0 0 0-1 1v7a1 1 0 0 0 1 1h2Z" fill="currentColor"></path></svg></div><div style="justify-content:center;display:flex;border-radius:99999px;width:2.5rem;height:2.5rem;cursor:pointer;background:;color:"><svg style="width:initial;height:initial" width="12" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M20.563 3.316A1.31 1.31 0 0 0 19.687 3h-1.688v9h1.688a1.31 1.31 0 0 0 1.312-1.077V4.077a1.31 1.31 0 0 0-.436-.761ZM16 12.788l-3.608 8.117A1.999 1.999 0 0 1 11 19v-4a1 1 0 0 0-1-1H4.328a1.002 1.002 0 0 1-1-1.15l1.38-9a1 1 0 0 1 1-.85h10.291v9.788ZM19.661 1a3.31 3.31 0 0 1 3.329 2.866c.006.044.01.09.01.134v7c0 .045-.004.09-.01.134A3.31 3.31 0 0 1 19.661 14h-2.012l-3.736 8.406a1 1 0 0 1-.914.594 4 4 0 0 1-4-4v-3H4.344a3 3 0 0 1-2.994-3.45l1.38-9A3.002 3.002 0 0 1 5.724 1h13.937Z" fill="currentColor"></path></svg></div></div><div class="Actions_uugI"><a target="_blank" class="Action_iBHd" href="https://github.com/apache/pulsar/issues/new?assignees=&labels=doc-required&projects=&template=doc.yml&title=%5BDoc%5D+">💡 Suggest changes</a><a target="_blank" class="Action_iBHd" href="https://github.com/apache/pulsar/discussions/new?category=q-a">🛟 Get support</a></div></div><ul class="table-of-contents table-of-contents__left-border"><li><a href="#key-concept" class="table-of-contents__link toc-highlight">Key concept</a><ul><li><a href="#transaction-coordinator" class="table-of-contents__link toc-highlight">Transaction coordinator</a></li><li><a href="#transaction-log" class="table-of-contents__link toc-highlight">Transaction log</a></li><li><a href="#transaction-buffer" class="table-of-contents__link toc-highlight">Transaction buffer</a></li><li><a href="#transaction-id" class="table-of-contents__link toc-highlight">Transaction ID</a></li><li><a href="#pending-acknowledge-state" class="table-of-contents__link toc-highlight">Pending acknowledge state</a></li></ul></li><li><a href="#data-flow" class="table-of-contents__link toc-highlight">Data flow</a><ul><li><a href="#1-begin-a-transaction" class="table-of-contents__link toc-highlight">1. Begin a transaction</a></li><li><a href="#2-publish-messages-with-a-transaction" class="table-of-contents__link toc-highlight">2. Publish messages with a transaction</a></li><li><a href="#3-acknowledge-messages-with-a-transaction" class="table-of-contents__link toc-highlight">3. Acknowledge messages with a transaction</a></li><li><a href="#4-end-a-transaction" class="table-of-contents__link toc-highlight">4. End a transaction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="footer__link-item">Foundation<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/events/current-event.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Events<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/security/">Security</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/policies/privacy.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Privacy<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/contact/">Contact</a></li></ul></div><div class="col footer__col"><div class="footer__title"></div><ul class="footer__items clean-list"><li class="footer__item"> |
| <div class="social-icons"> |
| <a target="_blank" href="https://communityinviter.com/apps/apache-pulsar/apache-pulsar" aria-label="Join the Apache Pulsar Slack workspace"> |
| <img alt="Slack logo" src="/img/slack-white.svg" width="26"> |
| </a> |
| <a target="_blank" href="https://github.com/apache/pulsar/" aria-label="View the Apache Pulsar project on GitHub"> |
| <img alt="GitHub logo" src="/img/github-white.svg" width="26"> |
| </a> |
| </div> |
| </li></ul></div></div><div class="footer__bottom text--center"><div class="margin-bottom--sm"><a class="footerLogoLink_BH7S" href="/"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--light_HNdA footer__logo"><img src="/img/pulsar-white.svg" alt="Pulsar Logo" class="themedImage_ToTc themedImage--dark_i4oU footer__logo"></a></div><div class="footer__copyright"> |
| <div> |
| <img class="footer-apache-logo" src="/img/feather-logo-white.svg" alt="" width="20"> |
| The Apache Software Foundation |
| </div> |
| <p>Apache Pulsar is available under the Apache License, version 2.0. Apache Pulsar is an open-source, distributed messaging and streaming platform built for the cloud.</p> |
| <p>Copyright © 2024 The Apache Software Foundation. All Rights Reserved. Apache, Pulsar, Apache Pulsar, and the Apache feather logo are trademarks or registered trademarks of The Apache Software Foundation.</p> |
| </div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.1d0ed2a7.js"></script> |
| <script src="/assets/js/main.e07a0c68.js"></script> |
| </body> |
| </html> |
