| <!doctype html> |
| <html lang="en" dir="ltr" class="mdx-wrapper mdx-page plugin-pages plugin-id-default"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.0"> |
| <title data-rh="true">BP-62 New API for batched reads | Apache BookKeeper</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://bookkeeper.apache.org/bps/BP-62-new-API-for-batched-reads"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docsearch:docusaurus_tag" content="default"><meta data-rh="true" property="og:title" content="BP-62 New API for batched reads | Apache BookKeeper"><meta data-rh="true" name="description" content="The BookKeeper client has provided an API [0] that allows for the efficient reading of entries [1] from bookies."><meta data-rh="true" property="og:description" content="The BookKeeper client has provided an API [0] that allows for the efficient reading of entries [1] from bookies."><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://bookkeeper.apache.org/bps/BP-62-new-API-for-batched-reads"><link data-rh="true" rel="alternate" href="https://bookkeeper.apache.org/bps/BP-62-new-API-for-batched-reads" hreflang="en"><link data-rh="true" rel="alternate" href="https://bookkeeper.apache.org/bps/BP-62-new-API-for-batched-reads" hreflang="x-default"><link rel="stylesheet" href="/assets/css/styles.49914aab.css"> |
| <link rel="preload" href="/assets/js/runtime~main.793d926f.js" as="script"> |
| <link rel="preload" href="/assets/js/main.c5d52852.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/bk-logo.svg" alt="Apache Bookkeeper" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/bk-logo.svg" alt="Apache Bookkeeper" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache BookKeeper</b></a><a class="navbar__item navbar__link" href="/docs/overview/">Documentation</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/community/mailing-lists">Mailing lists</a></li><li><a class="dropdown__link" href="/community/slack">Slack</a></li><li><a href="https://github.com/apache/bookkeeper/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Github issues<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/community/releases">Release management</a></li><li><a class="dropdown__link" href="/community/meeting">Community meetings</a></li><li><a class="dropdown__link" href="/community/contributing">Contribution guide</a></li><li><a class="dropdown__link" href="/community/coding-guide">Coding guide</a></li><li><a class="dropdown__link" href="/community/testing">Testing guide</a></li><li><a class="dropdown__link" href="/community/issue-report">Issue report guide</a></li><li><a class="dropdown__link" href="/community/release-guide">Release guide</a></li><li><a class="dropdown__link" href="/community/presentations">Presentations</a></li><li><a class="dropdown__link" href="/community/bookkeeper-proposals">BookKeeper proposals (BP)</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Project</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/project/who">Who are we?</a></li><li><a class="dropdown__link" href="/project/bylaws">Bylaws</a></li><li><a href="https://apache.org/licenses" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/project/privacy">Privacy policy</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/overview/">4.16.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/overview/">Next</a></li><li><a class="dropdown__link" href="/docs/overview/">4.16.5</a></li><li><a class="dropdown__link" href="/docs/4.15.5/overview/">4.15.5</a></li><li><a class="dropdown__link" href="/docs/4.14.8/overview/">4.14.8</a></li><li><a class="dropdown__link" href="/docs/4.13.0/overview/">4.13.0</a></li><li><a class="dropdown__link" href="/docs/4.12.1/overview/">4.12.1</a></li><li><a class="dropdown__link" href="/docs/4.11.1/overview/">4.11.1</a></li><li><a class="dropdown__link" href="/docs/4.10.0/overview/">4.10.0</a></li><li><a class="dropdown__link" href="/docs/4.9.2/overview/">4.9.2</a></li><li><a class="dropdown__link" href="/docs/4.8.2/overview/">4.8.2</a></li><li><a class="dropdown__link" href="/docs/4.7.3/overview/">4.7.3</a></li><li><a class="dropdown__link" href="/docs/4.6.2/overview/">4.6.2</a></li><li><a class="dropdown__link" href="/docs/4.5.1/overview/">4.5.1</a></li></ul></div><a class="navbar__item navbar__link" href="/releases">Download</a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><main class="container container--fluid margin-vert--lg"><div class="row mdxPageWrapper_j9I6"><div class="col col--8"><article><h1>BP-62 New API for batched reads</h1><h1>Motivation</h1><p>The BookKeeper client has provided an API <!-- -->[0]<!-- --> that allows for the efficient reading of entries <!-- -->[1]<!-- --> from bookies. |
| While users can specify a start and end entry ID for a ledger <!-- -->[2]<!-- --> according to the API's definition, |
| the reality is that the bookie server can only read a single entry at a time. Consequently, the BookKeeper client is |
| required to send a request for each entry to the bookies based on the user-specified start and end entry IDs. |
| This approach is highly inefficient and can result in significant CPU resource consumption, especially when reading a |
| large number of entries. The frequent RPC calls to the bookies also put a heavy burden on the bookie server, which must |
| process read requests from the IO thread and subsequently jump to the entry read thread pool. |
| <img loading="lazy" alt="single_read.png" src="/assets/images/single_read-41227c40cb1afd8bf5c1eddb1c78f287.png" width="2000" height="866" class="img_ev3q"></p><p>A more effective way to read entries is to use one RPC request to read multiple entries at once. This approach will |
| significantly reduce the number of RPC calls, thread jumps, and enqueue and dequeue operations of the thread pool. |
| <img loading="lazy" alt="batch_read.png" src="/assets/images/batch_read-6e348aa74b5b6d58019147a8007827d7.png" width="2000" height="901" class="img_ev3q"></p><p>To demonstrate this point, I have conducted a simple benchmark test using the BookKeeper perf tool to compare batch |
| and individual entry readings. The test case was repeated to read the data from the same ledger and to avoid the impact |
| of disk performance on test results, the entries were read from the cache since the batch read protocol change would not |
| introduce any disk read improvements. The results show that using the batched entry read API provides a remarkable |
| 10x performance improvement.</p><p>Here is the output of the BookKeeper perf tool with ensemble=1, write=1, and ack=1.</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Batch(100): Read 1000100 entries in 8904ms</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Batch(500): Read 1000500 entries in 12182ms</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Non-Batch: Read 1000130 entries in 199928ms</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The key difference with the batch read is that the BookKeeper client sends only 10,000 requests to the server |
| (compared to 1,000,000 in non-batch read), and the server sends 10,000 responses back to the client (versus 1,000,000 in non-batch read). |
| This approach reduces the number of RPC calls made and eliminates the need for the server to process a large number of |
| requests from the IO thread and the entry read thread pool, resulting in a significant improvement in performance.</p><p>Hence, the objective of this proposal is to enhance the performance of entry reading by introducing a batch entry reading protocol |
| that takes into account the expected count and size of entries.</p><ul><li><strong>Optimize entry reading performance:</strong> By reading multiple entries in a single RPC request, the network communication and RPC call overhead can be reduced, thereby optimizing the reading performance.</li><li><strong>Minimize CPU resource consumption:</strong> The aggregation of multiple entries into a single RPC request can help in reducing the number of requests and responses, which in turn can lower the CPU resource consumption.</li><li><strong>Streamline client code:</strong> The ability to read entries based on the anticipated count or size, such as Apache Pulsar's approach of calculating the start and end entry IDs for each read request based on the average size of past entries, can add unnecessary complexity to the implementation and can't guarantee reliable behavioral outcomes.</li></ul><h1>Public Interfaces</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bookkeeper-client-api">BookKeeper Client API<a href="#bookkeeper-client-api" class="hash-link" aria-label="Direct link to BookKeeper Client API" title="Direct link to BookKeeper Client API"></a></h2><ol><li>The new APIs will be added to BookieClient.java</li></ol><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">default void batchReadEntries(BookieId address, long ledgerId, long startEntryId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> int maxCount, long maxSize, BatchedReadEntryCallback cb, Object ctx,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> int flags) {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> batchReadEntries(address, ledgerId, startEntryId, maxCount, maxSize, cb, ctx, flags, null);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">default void batchReadEntries(BookieId address, long ledgerId, long startEntryId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> int maxCount, long maxSize, BatchedReadEntryCallback cb, Object ctx,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> int flags, byte[] masterKey) {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> batchReadEntries(address, ledgerId, startEntryId, maxCount, maxSize, cb, ctx, flags, masterKey, false);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> void batchReadEntries(BookieId address, long ledgerId, long startEntryId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> int maxCount, long maxSize, BatchedReadEntryCallback cb, Object ctx,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> int flags, byte[] masterKey, boolean allowFastFail);</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="2"><li>The new class BatchedReadEntryCallback will be added to BookkeeperInternalCallbacks.java</li></ol><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">public interface BatchedReadEntryCallback {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> void readEntriesComplete(int rc, long ledgerId, long startEntryId, ByteBufList bufList, Object ctx);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="3"><li>The new APIs will be added to ReadHandle.java</li></ol><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain"> default CompletableFuture<LedgerEntries> batchReadAsync(long startEntry, int maxCount, long maxSize);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">default LedgerEntries batchRead(long startEntry, int maxCount, long maxSize)</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> throws BKException, InterruptedException {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> return FutureUtils.result(batchReadAsync(startEntry, maxCount, maxSize), BKException.HANDLER);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> }</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="wire-protocol-changes">Wire protocol changes<a href="#wire-protocol-changes" class="hash-link" aria-label="Direct link to Wire protocol changes" title="Direct link to Wire protocol changes"></a></h2><p>In BookKeeper, the V2 protocol uses a custom encoding format. So we need to handle the data encoding and decoding. |
| The V3 protocol uses the ProtoBuf for encoding and decoding.(Not support now)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="v2-protocol">V2 Protocol<a href="#v2-protocol" class="hash-link" aria-label="Direct link to V2 Protocol" title="Direct link to V2 Protocol"></a></h3><p>The new command type <em><code>BATCH_READ_ENTRY</code></em> will be added to <a href="https://github.com/apache/bookkeeper/blob/master/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java" target="_blank" rel="noopener noreferrer">BookieProtocol.java</a></p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">byte BATCH_READ_ENTRY = 7;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The new command BatchedReadRequest will be added to BookieProtocol.java</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">class BatchedReadRequest extends ReadRequest {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long requestId;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> int maxCount;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long maxSize;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>And the command BatchedReadRequest will be encoded as</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">| 4 bytes (frame_size) | 4 bytes (header_size) | 4 bytes (packet_header) | 8 bytes (ledger_id) | 8 bytes (start_entry_id) | 8 bytes (request_id) | 4 bytes (max_count) | 8 bytes (max_size) |</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The new command <code>BatchedReadResponse</code> will be added to <a href="https://github.com/apache/bookkeeper/blob/master/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java" target="_blank" rel="noopener noreferrer">BookieProtocol.java</a></p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">class BatchedReadResponse extends Response implements ReferenceCounted {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> final long requestId;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> final ByteBufList data;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The new command <code>BatchedReadResponse</code> will be encoded as</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">| 4 bytes (frame_size) | 4 bytes (response_size) | 4 bytes (packet_header) | 4 bytes (error_code) | 8 bytes (ledger_id) | 8 bytes (start_entry_id) | 8 bytes (request_id) | 4 bytes (payload_size) | payload | 4 bytes (payload_size) | payload | ... |</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="v3-protocol">V3 Protocol<a href="#v3-protocol" class="hash-link" aria-label="Direct link to V3 Protocol" title="Direct link to V3 Protocol"></a></h3><p>The new Operation Type <em><code>BATCH_READ_ENTRY</code></em> will be added to the <a href="https://github.com/apache/bookkeeper/blob/master/bookkeeper-proto/src/main/proto/BookkeeperProtocol.proto#L55-L69" target="_blank" rel="noopener noreferrer">OperationType enum</a></p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">BATCH_READ_ENTRY = 12;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The new command will be added to <a href="https://github.com/apache/bookkeeper/blob/master/bookkeeper-proto/src/main/proto/BookkeeperProtocol.proto" target="_blank" rel="noopener noreferrer">BookkeeperProtocol.proto</a></p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">message BatchReadRequest {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> enum Flag {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> FENCE_LEDGER = 1;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ENTRY_PIGGYBACK = 2;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> optional Flag flag = 100;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> required int64 ledgerId = 1;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> // entryId will be -1 for reading the LAST_ADD_CONFIRMED entry.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> required int64 startEntryId = 2;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> required int32 maxCount = 3;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> required int64 maxSize = 4;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> // Used while fencing a ledger.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> optional bytes masterKey = 5;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> // Used for waiting on last add confirmed update</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> optional int64 previousLAC = 6;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> // Used as a timeout (in milliseconds) for the long polling request</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> optional int64 timeOut = 7;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">message BatchReadResponse {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> required StatusCode status = 1;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> required int64 ledgerId = 2;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> required int64 startEntryId = 3;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> repeated bytes body = 4;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> // Piggyback LAC</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> optional int64 maxLAC = 5;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> optional int64 lacUpdateTimestamp = 6;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Proposed Changes</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="loose-constraints">Loose constraints<a href="#loose-constraints" class="hash-link" aria-label="Direct link to Loose constraints" title="Direct link to Loose constraints"></a></h2><p>The definition of the batched read API does not strictly adhere to the maximum count and size constraints defined by the |
| request parameters. Rather, the API may return fewer entries than what was requested within the bounds of the established range. |
| For instance, if the caller requests to read 100 entries from the bookie server, the API may return any number of entries |
| up to and including 100. The API is designed as loose constraints for the following reasons</p><ol><li><strong>Ensemble change:</strong> The read entries may be distributed across different bookies after <em>Ensemble Change</em> <!-- -->[3]<!-- -->. |
| To cope with the impact of ensemble change, the BookKeeper client has to send multiple requests to different bookies when |
| reading entries and then merge the returned results to return to the user. If a request times out, the BookKeeper |
| client has to discard the other returned results and provide feedback to the caller. Compared to the complexity introduced for the client, |
| the benefits are almost negligible since ensemble change is not happening all the time.</li><li><strong>A < W:</strong> If <em>Ack Quorum</em> < <em>Write Quorum</em> <!-- -->[3]<!-- -->, some of the bookies may not have all the data that was successfully |
| written by the clients. Therefore, if a request is sent to bookies that do not have all the data the client successfully |
| wrote, the returned entries may also be missing some of the trailing data. We can have two ways to try to satisfy the |
| read size requirement |
| as much as possible |
| a. Discard the already read data and send the request to another bookie. |
| b. Read the missing parts from another bookie and then merge the results to return to the caller.</li></ol><p>But both options come with additional costs or added complexity for the bookkeeper client. And the API will not |
| return 0 entries or EntryNotFoundException to the caller if the first request entry ID is lower than the <em>LAC</em> <!-- -->[3]<!-- -->. |
| The bookie client would retry to another bookie to read the entries if the requested bookie returned EntryNotFoundException.</p><ol start="3"><li><strong>Exceeds the Netty’s frame size:</strong> The bookie server cannot send data larger than the Netty’s frame size.</li></ol><p>In conclusion, defining the maximum read entries count and size in the new API as loose constraints would simplify the client. |
| For the API caller, it is only necessary to decide which entry to read from the entries that have already been read without |
| worrying about whether the read entries meet the expected entries count or size of the request.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="not-supporting-striped-ledger">Not supporting striped Ledger<a href="#not-supporting-striped-ledger" class="hash-link" aria-label="Direct link to Not supporting striped Ledger" title="Direct link to Not supporting striped Ledger"></a></h2><p>Regarding what bookkeeper striped writes are, please refer to the section of <em>Layer 2 - Logical Storage Model of the post</em> <!-- -->[4]<!-- -->. |
| As no single bookie can have the complete set of written data, it poses a significant challenge to perform batch reads. |
| The client has to read from multiple bookies and merge the results to return to the caller. One possibility is the API |
| can be defined as reading multiple entry IDs. The caller provides a list of entry IDs. The bookie client splits into |
| multiple requests based on provided entry IDs for each bookie and then merges the returned results from multiple bookies |
| before returning them to the caller. Considering the following reasons, we should not define the API like this.</p><ol><li>The implementation of the client becomes more complicated.</li><li>From my experience, striped writing is not commonly used. We should not add overhead to common scenarios for less |
| frequently used ones, especially if it means that the request body containing the entry ID list becomes larger. |
| This mainly refers to the fact that including an entry ID list in the request will inflate the network transmission.</li><li>The bookie server must return the data according to the defined entry IDs. If one entry is missed on the bookie, |
| the bookie must return EntryNotFoundException to the bookie client so that the bookie client can request another bookie. |
| Otherwise, the API definition will become confusing.</li></ol><p>So, the bookie client will fall back to the current way of reading a single entry at a time to a bookie and merging all |
| the results from the bookies if the <em>Ensemble size</em> is greater than the <em>Write Quorum</em>.</p><p>For the fallback, we have two options:</p><ul><li>Use the batch-read API to read entries, but bookie servers will return one entry in each read request. After the entry |
| is returned, the client will send the next read request to the bookie servers. This way can unify caller API to batch-read |
| but will introduce a huge read performance impact due to the next read request won’t be sent to the bookies until the |
| previous read response is returned.</li><li>Provide a fallback extended batch-read API. If the ledger is stripped, the batch-read API will automatically fall back |
| to the single-read API internally. The fallback batch-read API is an extension of the batch-read API, and it can fall |
| back to single-read API internally on demand.</li></ul><p>We will choose the fallback batch-read API solution. This solution can provide the simplest way to support reading stripped |
| ledgers in the unified batch-read API and users don’t need to care about the compatibility issues.</p><h1>Compatibility</h1><p>Considering the introduction of wire protocol changes in the proposal, there is a possibility of compatibility issues arising |
| if the client is upgraded without upgrading the server. This could result in potential errors and unexpected behavior. |
| To prevent any losses for the users, there are two options available. However, it is important to note that this is a newly |
| introduced API and not a modification of any existing APIs. The compatibility issue being referred to is specifically |
| related to the definition of the new API when used with an old version bookie server. It is worth mentioning that the |
| proposed changes will not break any existing APIs or alter the behavior of any existing APIs.</p><ol><li><strong>Default batch-read API returns error codes directly.</strong> The BookKeeper protocol has established the "invalid request type" |
| error <!-- -->[5]<!-- --> to handle cases where a client sends a request to a bookie server that does not support batch reading. Consequently, |
| the client will receive a "NotSupport" error and must decide whether to switch to single-entry read mode. While this approach |
| offers more deterministic behavior for the new API, ensuring that callers are informed of the ultimate behavior whether or not |
| the server supports it, it does require all callers to handle the issue of unsupported API.</li><li><strong>Fallback-supported batch-read API will fallback to single-entry read automatically.</strong> By letting the bookie client |
| handle the issue of server support for the batch read protocol, the batch read API will not fail when the server does not |
| support it. This decouples the client batch API from the server protocol, meaning that users do not need to worry about |
| whether the bookie server supports batch reading or not. However, this may be confusing for users as they may not receive |
| a clear indication that the server does not support the new API. Users expect to achieve better performance by using the new API, |
| but if the server has not been upgraded, the new API may not perform as expected.</li></ol><p>In order to make batch-read API definition and behavior clear, we provide the batch-read API and fallback-support batch-read API. |
| The default batch-read API returns error codes to the user directly and let the user determine what action to do. The fallback-supported |
| batch-read API will handle the unsupported cases internally and the user won’t receive any alerts if the bookie server does not support |
| batch-read API or reading to the stripped ledgers.</p><p>The proposal aims to define the behavior of the new API through the <strong>second approach</strong>, primarily due to the following reasons:</p><ol><li>The responsibility of handling fallback to single-entry read mode for Ledgers with stripped writing should belong to |
| the bookie client, as stripped writing is not part of the public API of BookKeeper. Therefore, it is necessary to maintain |
| consistent behavior across all BookKeeper clients.</li><li>To avoid users from making redundant efforts to handle protocol compatibility, the client should handle the issue of |
| unsupported API rather than leave it to the users.</li><li>If applications such as Pulsar want to migrate from single-read API to batch-read API, they need to change the whole |
| read logic because the batch read API is different from the single read API, because the constraints of the batch read API |
| are loose, and single read API is strict. If the applications switched from single-read API to batch-read API by changing |
| the code logic, we need a way to switch back to single-read API. One solution is to maintain both single-read API and |
| batch-read API on the application side, and another solution is to provide a flag on the BookKeeper client side, and fallback |
| to single-read API if the flag is enabled. We prefer to control it on the BookKeeper client side because it will simplify the |
| application logic a lot when switching from single-read API to bath-read API.</li></ol><p>In conclusion, detailed documentation should be provided to explain the API's behavior, particularly in cases where it |
| falls back to single-item read mode. This will help users set reasonable expectations for the new API and avoid confusion.</p><h1>Operability Considerations</h1><p>Even if users have upgraded the BookKeeper server and client, modified their code to use the new API for better read |
| performance. However, the new API may also introduce new risks. It could be an increased resource consumption in specific |
| scenarios or a bug that causes the API to malfunction in certain cases. Therefore, users need to have a quick option to |
| go back to the old way when problems arise, rather than only being able to resolve them through rollbacking the application.</p><p>The fallback-supported batch-read API provided a flag to fall back to the old behavior. We control the flag on the |
| BookKeeper client side can simplify the application logic a lot when switching from single-read API to bath-read API.</p><h1>Security Considerations</h1><p>No security issues that we need to consider for this proposal. It will follow the existing security pattern.</p><h1>Observability</h1><p>To increase visibility into the new functionality of the bookie server, we need to add the following metrics to the bookie server</p><table><thead><tr><th>Indicator</th><th>Type</th><th>Description</th></tr></thead><tbody><tr><td>bookkeeper_server_BATCH_READ_ENTRY_REQUEST</td><td>Histogram</td><td>The histogram of the batch read requests</td></tr><tr><td>latency buckets: <!-- -->[0-5]<!-- -->, <!-- -->[5-10]<!-- -->, <!-- -->[10, 20]<!-- -->, <!-- -->[20-50]<!-- -->, <!-- -->[50-100]<!-- -->, <!-- -->[100-200]<!-- -->,<!-- -->[200-500]<!-- -->,<!-- -->[500-1000]<!-- -->, <!-- -->[1000-3000]<!-- -->, <!-- -->[>3000]</td><td></td><td></td></tr><tr><td>bookkeeper_server_BATCH_READ_ENTRY_RESPONSE_SIZE</td><td>Histogram</td><td>The histogram of the batch read response size</td></tr><tr><td>Size buckets: <!-- -->[0-128]<!-- -->, <!-- -->[128-512]<!-- -->,<!-- -->[512-1k]<!-- -->,<!-- -->[1k-2k]<!-- -->,<!-- -->[2k-4k]<!-- -->,<!-- -->[4k,16k]<!-- -->,<!-- -->[16k-128k]<!-- -->,<!-- -->[128k-1MB]<!-- -->, <!-- -->[> 1MB]</td><td></td><td></td></tr></tbody></table><h1>References</h1><p>[0]<!-- --> <a href="https://bookkeeper.apache.org/docs/api/ledger-api#reading-entries-from-ledgers" target="_blank" rel="noopener noreferrer">https://bookkeeper.apache.org/docs/api/ledger-api#reading-entries-from-ledgers</a> |
| <!-- -->[1]<!-- --> <a href="https://bookkeeper.apache.org/docs/getting-started/concepts#entries" target="_blank" rel="noopener noreferrer">https://bookkeeper.apache.org/docs/getting-started/concepts#entries</a> |
| <!-- -->[2]<!-- --> <a href="https://bookkeeper.apache.org/docs/getting-started/concepts#ledgers" target="_blank" rel="noopener noreferrer">https://bookkeeper.apache.org/docs/getting-started/concepts#ledgers</a> |
| <!-- -->[3]<!-- --> <a href="https://jack-vanlightly.com/blog/2021/12/7/tweaking-the-bookkeeper-protocol-guaranteeing-write-quorum" target="_blank" rel="noopener noreferrer">https://jack-vanlightly.com/blog/2021/12/7/tweaking-the-bookkeeper-protocol-guaranteeing-write-quorum</a> |
| <!-- -->[4]<!-- --> <a href="https://jack-vanlightly.com/blog/2018/10/2/understanding-how-apache-pulsar-works" target="_blank" rel="noopener noreferrer">https://jack-vanlightly.com/blog/2018/10/2/understanding-how-apache-pulsar-works</a> |
| <!-- -->[5]<!-- --> <a href="https://github.com/apache/bookkeeper/blob/master/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java#L151" target="_blank" rel="noopener noreferrer">https://github.com/apache/bookkeeper/blob/master/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java#L151</a></p></article></div><div class="col col--2"><div class="tableOfContents_bqdL thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#bookkeeper-client-api" class="table-of-contents__link toc-highlight">BookKeeper Client API</a></li><li><a href="#wire-protocol-changes" class="table-of-contents__link toc-highlight">Wire protocol changes</a><ul><li><a href="#v2-protocol" class="table-of-contents__link toc-highlight">V2 Protocol</a></li><li><a href="#v3-protocol" class="table-of-contents__link toc-highlight">V3 Protocol</a></li></ul></li><li><a href="#loose-constraints" class="table-of-contents__link toc-highlight">Loose constraints</a></li><li><a href="#not-supporting-striped-ledger" class="table-of-contents__link toc-highlight">Not supporting striped Ledger</a></li></ul></div></div></div></main></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Documentation</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/overview">Overview</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/getting-started/installation">Getting started</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/deployment/manual">Deployment</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/admin/bookies">Administration</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/api/overview">API</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/security/overview">Security</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/development/protocol">Development</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/reference/config">Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/community/mailing-lists">Mailing lists</a></li><li class="footer__item"><a class="footer__link-item" href="/community/slack">Slack</a></li><li class="footer__item"><a href="https://github.com/apache/bookkeeper" target="_blank" rel="noopener noreferrer" class="footer__link-item">Github<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://twitter.com/asfbookkeeper" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">Project</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/project/who">Who are we?</a></li><li class="footer__item"><a class="footer__link-item" href="/project/bylaws">Bylaws</a></li><li class="footer__item"><a href="https://apache.org/licenses" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/project/privacy">Privacy policy</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright"><footer class="footer"> |
| <div class="container"> |
| <div class="content has-text-centered"> |
| <p> |
| Copyright © 2016 - 2024 <a href="https://www.apache.org/">The Apache Software Foundation</a>,<br> licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, version 2.0</a>. |
| </p> |
| <p> |
| Apache BookKeeper, BookKeeper®, Apache®, the Apache feature logo, and the Apache BookKeeper logo are either registered trademarks or trademarks of The Apache Software Foundation. |
| </p> |
| </div> |
| </div> |
| </footer> |
| </div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.793d926f.js"></script> |
| <script src="/assets/js/main.c5d52852.js"></script> |
| </body> |
| </html> |
