| <!doctype html> |
| <html lang="en" dir="ltr" class="mdx-wrapper mdx-page plugin-pages plugin-id-default"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.0"> |
| <title data-rh="true">BP-35: 128 bits support | Apache BookKeeper</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://bookkeeper.apache.org/bps/BP-35-128-bits-support"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docsearch:docusaurus_tag" content="default"><meta data-rh="true" property="og:title" content="BP-35: 128 bits support | Apache BookKeeper"><meta data-rh="true" name="description" content="Motivation"><meta data-rh="true" property="og:description" content="Motivation"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://bookkeeper.apache.org/bps/BP-35-128-bits-support"><link data-rh="true" rel="alternate" href="https://bookkeeper.apache.org/bps/BP-35-128-bits-support" hreflang="en"><link data-rh="true" rel="alternate" href="https://bookkeeper.apache.org/bps/BP-35-128-bits-support" hreflang="x-default"><link rel="stylesheet" href="/assets/css/styles.49914aab.css"> |
| <link rel="preload" href="/assets/js/runtime~main.1c369ecb.js" as="script"> |
| <link rel="preload" href="/assets/js/main.812b2dbb.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/bk-logo.svg" alt="Apache Bookkeeper" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/bk-logo.svg" alt="Apache Bookkeeper" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache BookKeeper</b></a><a class="navbar__item navbar__link" href="/docs/overview/">Documentation</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/community/mailing-lists">Mailing lists</a></li><li><a class="dropdown__link" href="/community/slack">Slack</a></li><li><a href="https://github.com/apache/bookkeeper/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Github issues<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/community/releases">Release management</a></li><li><a class="dropdown__link" href="/community/meeting">Community meetings</a></li><li><a class="dropdown__link" href="/community/contributing">Contribution guide</a></li><li><a class="dropdown__link" href="/community/coding-guide">Coding guide</a></li><li><a class="dropdown__link" href="/community/testing">Testing guide</a></li><li><a class="dropdown__link" href="/community/issue-report">Issue report guide</a></li><li><a class="dropdown__link" href="/community/release-guide">Release guide</a></li><li><a class="dropdown__link" href="/community/presentations">Presentations</a></li><li><a class="dropdown__link" href="/community/bookkeeper-proposals">BookKeeper proposals (BP)</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Project</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/project/who">Who are we?</a></li><li><a class="dropdown__link" href="/project/bylaws">Bylaws</a></li><li><a href="https://apache.org/licenses" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/project/privacy">Privacy policy</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/overview/">4.16.5</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/overview/">Next</a></li><li><a class="dropdown__link" href="/docs/overview/">4.16.5</a></li><li><a class="dropdown__link" href="/docs/4.15.5/overview/">4.15.5</a></li><li><a class="dropdown__link" href="/docs/4.14.8/overview/">4.14.8</a></li><li><a class="dropdown__link" href="/docs/4.13.0/overview/">4.13.0</a></li><li><a class="dropdown__link" href="/docs/4.12.1/overview/">4.12.1</a></li><li><a class="dropdown__link" href="/docs/4.11.1/overview/">4.11.1</a></li><li><a class="dropdown__link" href="/docs/4.10.0/overview/">4.10.0</a></li><li><a class="dropdown__link" href="/docs/4.9.2/overview/">4.9.2</a></li><li><a class="dropdown__link" href="/docs/4.8.2/overview/">4.8.2</a></li><li><a class="dropdown__link" href="/docs/4.7.3/overview/">4.7.3</a></li><li><a class="dropdown__link" href="/docs/4.6.2/overview/">4.6.2</a></li><li><a class="dropdown__link" href="/docs/4.5.1/overview/">4.5.1</a></li></ul></div><a class="navbar__item navbar__link" href="/releases">Download</a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><main class="container container--fluid margin-vert--lg"><div class="row mdxPageWrapper_j9I6"><div class="col col--8"><article><h1>BP-35: 128 bits support</h1><h3 class="anchor anchorWithStickyNavbar_LWe7" id="motivation">Motivation<a href="#motivation" class="hash-link" aria-label="Direct link to Motivation" title="Direct link to Motivation"></a></h3><p>BookKeeper coordinates with a metadata store to generate a cluster wide <code>ledgerId</code>. |
| Currently this is a signed <code>64 bit</code> number (effectively 63 bits). This method works |
| great because we have a centralized metadata store for coordinating the id generation. |
| However this method may not scale as the cluster size and number of ledgers grow.</p><p><a href="https://en.wikipedia.org/wiki/Globally_unique_identifier" target="_blank" rel="noopener noreferrer">Universally unique identifier - Wikipedia</a> |
| is a preferred way to generate decentralized globally unique IDs and it takes <code>128 bits</code>. |
| This method can scale well as it doesn't need a centralized coordination. </p><p>This BP proposes the changes for increasing ledger id from <code>63 bits</code> to <code>128 bits</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="128-bits">128 bits<a href="#128-bits" class="hash-link" aria-label="Direct link to 128 bits" title="Direct link to 128 bits"></a></h3><p>Since there is no native support for <code>128 bits</code> in both Java and |
| <a href="https://github.com/google/protobuf/issues/2180" target="_blank" rel="noopener noreferrer">Protobuf</a>, we have to break <code>128 bits</code> |
| into 2 <code>64 bits</code> numbers for representing the <code>128 bits</code> id:</p><ul><li>ledger-id-msb: the most significant 64 bits, bit 64 - 127</li><li>ledger-id-lsb: the least significant 64 bits, bit 0 - 63</li></ul><p>For backward compatibility, the <code>ledger-id-lsb</code> is the current <code>64 bits</code> ledger-id. |
| The <code>ledger-id-msb</code> will be added as a new field in both API and protocol. </p><p>I am proposing calling <code>ledger-id-msb</code> as <code>ledger-scope-id</code>. So the current 64bits <code>ledgerId</code> and |
| the newly introduced 64bits <code>ledgerScopeId</code> together will be forming the new <code>128 bits</code> ledger id.</p><p>The default <code>ledgerScopeId</code> is <code>0</code>. That means any ledgers created prior to this change are allocated |
| under scope <code>0</code>. Hence it maintains backward compatibility during upgrade. </p><p>The combination of <code>ledgerScopeId</code> and <code>ledgerId</code> forms the <code>128 bits</code> ledger id. We can introduce a |
| hex representation of this <code>128 bits</code> ledger id - <code>ledgerQualifiedName</code> . This <code>ledgerQualifiedName</code> can |
| be useful for CLI tooling, REST api and troubleshooting purpose. The API internally can convert |
| <code>ledgerQualifiedName</code> to <code>ledgerScopeId</code> and <code>ledgerId</code>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="public-interfaces">Public Interfaces<a href="#public-interfaces" class="hash-link" aria-label="Direct link to Public Interfaces" title="Direct link to Public Interfaces"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="api-change">API Change<a href="#api-change" class="hash-link" aria-label="Direct link to API Change" title="Direct link to API Change"></a></h4><p>The API will be introducing <code>ledgerScopeId</code> across the interfaces. This field will be optional and default to <code>0</code>. </p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="handle">Handle<a href="#handle" class="hash-link" aria-label="Direct link to Handle" title="Direct link to Handle"></a></h5><p>Introduce a new method <code>getScopeId</code> for representing the scope id (the most significant <code>128 bits</code> ledger id).</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">public interface Handle extends AutoCloseable {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> * Return the ledger scope id. The most significant 64 bits of 128 bits.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long getScopeId();</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> * Return the ledger id. The least significant 64 bits of 128 bits.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> */ </span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long getId();</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="create-ledgeradv">Create LedgerAdv<a href="#create-ledgeradv" class="hash-link" aria-label="Direct link to Create LedgerAdv" title="Direct link to Create LedgerAdv"></a></h5><p>Introduce a new method <code>withLedgerScopeId</code> in <code>CreateAdvBuilder</code> for providing <code>scopeId</code> |
| (the most significant 64 bits for 128 bits ledger id) on creating a ledger.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">public interface CreateAdvBuilder extends OpBuilder<WriteHandle> {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> * Set the scope id for the newly created ledger.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> * If no explicit scopeId is passed, the new ledger</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> * will be created under scope `0`.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> CreateAdvBuilder withLedgerScopeId(long scopeId); </span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="open-ledger">Open Ledger<a href="#open-ledger" class="hash-link" aria-label="Direct link to Open Ledger" title="Direct link to Open Ledger"></a></h5><p>Introduce a new method <code>withLedgerScopeId</code> in <code>OpenBuilder</code> for providing <code>scopeId</code> |
| (the most significant 64 bits for 128 bits ledger id) on opening a ledger.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">public interface OpenBuilder extends OpBuilder<ReadHandle> {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> * Set the scope id of the ledger to open.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> OpenBuilder withLedgerScopeId(long scopeId);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h5 class="anchor anchorWithStickyNavbar_LWe7" id="delete-ledger">Delete Ledger<a href="#delete-ledger" class="hash-link" aria-label="Direct link to Delete Ledger" title="Direct link to Delete Ledger"></a></h5><p>Introduce a new method <code>withLedgerScopeId</code> in <code>DeleteBuilder</code> for providing <code>scopeId</code> |
| (the most significant 64 bits for 128 bits ledger id) on deleting a ledger.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">public interface DeleteBuilder extends OpBuilder<Void> {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> /**</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> * Set the scope id of the ledger to delete.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> */</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> DeleteBuilder withLedgerScopeId(long scopeId);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="cli">CLI<a href="#cli" class="hash-link" aria-label="Direct link to CLI" title="Direct link to CLI"></a></h4><p>All BookKeeper CLI tools will be updated with additional option <code>—ledger-scope-id</code>. |
| Optionally we can add option <code>—ledger-qualified-name</code> (the hex representation of 128 bits). |
| Internally all the CLI tools will convert ledger qualified name to <code>ledgerId</code> and <code>ledgerScopeId</code>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="rest">REST<a href="#rest" class="hash-link" aria-label="Direct link to REST" title="Direct link to REST"></a></h4><ol><li>All ledger related endpoints will be adding a new parameter <code>ledger_scope_id</code>. </li><li><code>ListLedgerService</code> only supports listing ledgers under a given ledger scope id. |
| If <code>ledger_scope_id</code> is missing, it will be listing ledgers under scope <code>0</code>.</li></ol><h4 class="anchor anchorWithStickyNavbar_LWe7" id="wire-protocol">Wire Protocol<a href="#wire-protocol" class="hash-link" aria-label="Direct link to Wire Protocol" title="Direct link to Wire Protocol"></a></h4><blockquote><p>There will be no plan for supporting 128 bits in v2 protocol, due to the limitation in v2 protocol. |
| So any operations in v2 protocol with scope id not equal to 0 will be failed immediately with |
| <code>ILLEGAL_OP</code> exceptions.</p></blockquote><p>All the request and response messages will be adding an optional field <code>optional int64 ledgerScopeId</code>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="entry-format">Entry Format<a href="#entry-format" class="hash-link" aria-label="Direct link to Entry Format" title="Direct link to Entry Format"></a></h4><p>Currently all the entries written to bookies are encoded in a certain format, including <code>metadata</code>, |
| <code>digest code</code> and <code>payload</code>. The entry format is not <em>versioned</em>.</p><p>In order to support adding another field <code>ledgerScopeId</code> in the <code>metadata</code> section, we are introducing |
| <code>version</code> in the entry format.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="entry-format-v1">Entry Format V1<a href="#entry-format-v1" class="hash-link" aria-label="Direct link to Entry Format V1" title="Direct link to Entry Format V1"></a></h5><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Entry Format V1</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">===============</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">--- header ---</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">0</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">7</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Ledger ID</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">8</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">15</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Entry ID</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">16</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">23</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> LastAddConfirmed</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">24</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">31</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Length</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">--- digest ---</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">32</span><span class="token plain"> - (</span><span class="token number" style="color:#36acaa">32</span><span class="token plain"> + x - </span><span class="token number" style="color:#36acaa">1</span><span class="token plain">)) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Digest Code (e.g. CRC32)</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">--- payload ---</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes ((</span><span class="token number" style="color:#36acaa">32</span><span class="token plain"> + x) - ) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Payload</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p><code>x</code> is the length of digest code.</p></blockquote><blockquote><p> Prior to introducing <code>ledgerScopeId</code>, ledgerId is assumed to be a positive value.</p></blockquote><h5 class="anchor anchorWithStickyNavbar_LWe7" id="entry-format-v2">Entry Format V2<a href="#entry-format-v2" class="hash-link" aria-label="Direct link to Entry Format V2" title="Direct link to Entry Format V2"></a></h5><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Entry Format V2</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">===============</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">--- header ---</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">0</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">7</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Metadata Flags</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">8</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">15</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Ledger Scope ID</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">16</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">23</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Ledger ID</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">24</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">31</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Entry ID</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">32</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">39</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> LastAddConfirmed</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">40</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">47</span><span class="token plain">) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Length</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">--- digest ---</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes (</span><span class="token number" style="color:#36acaa">37</span><span class="token plain"> - (</span><span class="token number" style="color:#36acaa">37</span><span class="token plain"> + x - </span><span class="token number" style="color:#36acaa">1</span><span class="token plain">)) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Digest Code (e.g. CRC32)</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">--- payload ---</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bytes ((</span><span class="token number" style="color:#36acaa">37</span><span class="token plain"> + x) - ) </span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> Payload</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p><code>x</code> is the length of digest code.</p></blockquote><h6 class="anchor anchorWithStickyNavbar_LWe7" id="metadata-flags">Metadata Flags<a href="#metadata-flags" class="hash-link" aria-label="Direct link to Metadata Flags" title="Direct link to Metadata Flags"></a></h6><div class="language-json codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-json codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Metadata</span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">1</span><span class="token plain"> Bytes (Long)</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">------------------------</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">0x </span><span class="token number" style="color:#36acaa">0</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> |__| </span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> |</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> version</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">----</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bit </span><span class="token number" style="color:#36acaa">0</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">3</span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> digest type (e.g. CRC32</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"> CRC32C and such)</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">Bit </span><span class="token number" style="color:#36acaa">4</span><span class="token plain"> - </span><span class="token number" style="color:#36acaa">7</span><span class="token operator" style="color:#393A34">:</span><span class="token plain"> version</span><span class="token punctuation" style="color:#393A34">,</span><span class="token plain"> the most significant bit of this byte will be always set to </span><span class="token number" style="color:#36acaa">1</span><span class="token plain">.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">it will be used for differentiating entry format v1 and v2.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We are setting the most significant bit to be <code>1</code>. So the first byte in entry v2 will |
| be a negative value, which can be used for differentiating entry format v1 and v2. |
| The version will be encoded into the first byte. The version will be used for describing |
| the entry format.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="decoding-entry">Decoding Entry<a href="#decoding-entry" class="hash-link" aria-label="Direct link to Decoding Entry" title="Direct link to Decoding Entry"></a></h5><p>The pseudo code for decoding an entry will be described as followings:</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">ByteBuf entry = ...;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">int metadataFlags = entry.getByte();</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">if (metadataFlags <= 128) { // the entry is encoded in v1 format</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> // decoding the entry in v1 format</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> ...</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">} else {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> // decoding the entry in v2 format</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="bookie-storage">Bookie Storage<a href="#bookie-storage" class="hash-link" aria-label="Direct link to Bookie Storage" title="Direct link to Bookie Storage"></a></h4><h5 class="anchor anchorWithStickyNavbar_LWe7" id="journal">Journal<a href="#journal" class="hash-link" aria-label="Direct link to Journal" title="Direct link to Journal"></a></h5><p>A new method should be added in journal <code>WriteCallback</code> to handle <code>ledgerScopeId</code>.</p><div class="language-java codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-java codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">public interface WriteCallback {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> void writeComplete(int rc,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long ledgerScopeId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long ledgerId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long entryId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> BookieSocketAddress addr,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> Object ctx);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> default void writeComplete(int rc,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long ledgerId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> long entryId,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> BookieSocketAddress addr,</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> Object ctx) {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> writeComplete(rc, 0L, ledgerId, entryId, addr, ctx);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The journal should be changed to be able to retrieve <code>ledgerScopeId</code> from the entry |
| payload based on <a href="#entry-format">Entry Format</a>.</p><h5 class="anchor anchorWithStickyNavbar_LWe7" id="ledger-storage">Ledger Storage<a href="#ledger-storage" class="hash-link" aria-label="Direct link to Ledger Storage" title="Direct link to Ledger Storage"></a></h5><h6 class="anchor anchorWithStickyNavbar_LWe7" id="entrylogger">EntryLogger<a href="#entrylogger" class="hash-link" aria-label="Direct link to EntryLogger" title="Direct link to EntryLogger"></a></h6><ol><li>Methods in <code>EntryLogger</code> should be able to accept <code>ledgerScopeId</code> as a parameter.</li><li>EntryLogger should be updated to retrieve <code>ledgerScopeId</code> from the entry payload |
| based on <a href="#entry-format">Entry Format</a>.</li></ol><h6 class="anchor anchorWithStickyNavbar_LWe7" id="entrymemtable">EntryMemTable<a href="#entrymemtable" class="hash-link" aria-label="Direct link to EntryMemTable" title="Direct link to EntryMemTable"></a></h6><p><code>ledgerScopeId</code> should be added as part of <code>EntryKey</code>.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="indexpersistencemgr">IndexPersistenceMgr<a href="#indexpersistencemgr" class="hash-link" aria-label="Direct link to IndexPersistenceMgr" title="Direct link to IndexPersistenceMgr"></a></h6><p>Currently the ledger index files (64 bits) are stored into 2-level-hirechicy |
| directories - <code><msb-32bits-hex>/<lsb-32bits-hex>/<ledger-id-hex>.idx</code>.</p><p>If <code>ledgerScopeId</code> is 0, it will be using existing scheme for storing and retrieving |
| ledger index files.</p><p>If <code>ledgerScopeId</code> is not 0, that means the ledgers are produced by new clients that |
| support 128-bits, those ledgers will be stored in a 4-level-hirechicy |
| directories - |
| <code><msb-32bits-hex-ledger-scope-id>/<lsb-32bits-hex-ledger-scope-id>/<msb-32bits-hex-ledger-id>/<lsb-32bits-hex-ledger-id></code>.</p><p>All the file info caches should be updated to use <code><ledgerScopeId, ledgerId></code> |
| as index keys.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="indexinmempagemgr">IndexInMemPageMgr<a href="#indexinmempagemgr" class="hash-link" aria-label="Direct link to IndexInMemPageMgr" title="Direct link to IndexInMemPageMgr"></a></h6><p>The LRU pages map will be updated to use <code><ledgerScopeId, ledgerId></code> as index |
| keys.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="dbledgerstorage">DBLedgerStorage<a href="#dbledgerstorage" class="hash-link" aria-label="Direct link to DBLedgerStorage" title="Direct link to DBLedgerStorage"></a></h6><p>Currently DBLedgerStorage use <code><ledgerId, entryId></code> as the index key for indexing entry |
| locations for each entry.</p><p>Similar as <code>SortedLedgerStorage</code> and <code>InterleavedLedgerStorage</code>, for ledgers whose |
| <code>ledgerScopeId</code> is 0, they will be using existing scheme for storing their entry locations.</p><p>For ledgers whose <code>ledgerScopeId</code> is not 0, they will be stored in a new rocksdb, |
| whose index key will be <code><ledgerScopeId, ledgerId, entryId></code>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="metadata-store">Metadata Store<a href="#metadata-store" class="hash-link" aria-label="Direct link to Metadata Store" title="Direct link to Metadata Store"></a></h4><h5 class="anchor anchorWithStickyNavbar_LWe7" id="ledgermanager">LedgerManager<a href="#ledgermanager" class="hash-link" aria-label="Direct link to LedgerManager" title="Direct link to LedgerManager"></a></h5><p>All the interfaces should be updated with accepting <code>ledgerScopeId</code>.</p><p>The actual implementation should decide how to store metadata |
| for <code><ledgerScopeId, ledgerId></code>. </p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="zookeeper-ledger-manager">ZooKeeper Ledger Manager<a href="#zookeeper-ledger-manager" class="hash-link" aria-label="Direct link to ZooKeeper Ledger Manager" title="Direct link to ZooKeeper Ledger Manager"></a></h6><p>We need to introduce a LongLongHierchicalLedgerManager for storing metadata |
| indexing by <code><ledgerScopeId, ledgerId></code>.</p><p>If <code>ledgerScopeId</code> is 0, then it will be falling back to <code>LongHierachicalLedgerManager</code>. |
| So no behavior is changed.</p><p>If <code>ledgerScopeId</code> is not 0, those ledgers will be indexed in new hierarchy |
| (possible under a different znode).</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="ledger-id-generation">Ledger ID generation<a href="#ledger-id-generation" class="hash-link" aria-label="Direct link to Ledger ID generation" title="Direct link to Ledger ID generation"></a></h6><p>When upgrading from 64bit to 128bits, we probably don't need any centralized mechanism |
| for generating ledger id. It can be implemented using UUID generation.</p><p>Especially since we are supporting 128bits by introducing <code>ledgerScopeId</code>. That means |
| application of bookkeeper can decide its own way for generating their <code>scopeId</code>. |
| An application or even bookkeeper client can generate its ledgerId using UUID generation, |
| then breaks the 128 bits UUID into two parts, one serves as <code>ledgerScopeId</code> and the other |
| one serves as <code>ledgerId</code>.</p><h6 class="anchor anchorWithStickyNavbar_LWe7" id="etcd">Etcd<a href="#etcd" class="hash-link" aria-label="Direct link to Etcd" title="Direct link to Etcd"></a></h6><p>Since Etcd has a better key/value presentation, we can basically just combine |
| <code><ledgerScopeId, ledgerId></code> as the index key for storing ledger metadata in Etcd. |
| Nothing is needed for special consideration.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="performance-concerns">Performance Concerns<a href="#performance-concerns" class="hash-link" aria-label="Direct link to Performance Concerns" title="Direct link to Performance Concerns"></a></h3><p>There shouldn't be any performance difference when not using 128 bit ledger id |
| (<code>ledgerScopeId</code> is omitted).</p><p>Performance concerns can be arised in following areas:</p><ul><li><strong>Wire Protocol</strong>: additional 9 bytes will be added per entry, one byte for version |
| and 8 bytes for the msb of 128 bit ledger id</li><li><strong>Journal</strong>: additional 9 bytes will be added per entry (same as wire protocol). </li><li><strong>EntryLogger</strong>: additional 9 bytes will be added per entry (same as wire protocol)</li><li><strong>Memtable</strong>: additional 8 bytes will be added per indexed entry.</li><li><strong>FileInfo</strong>: there is no change to the index file format itself.</li><li><strong>IndexPersistenceManager</strong>: Files will be organized in more directory hierarchy. |
| It shouldn't be a big deal. </li><li><strong>IndexInMemoryManager (LedgerCache)</strong>: additional 8 bytes per index page.</li><li><strong>DbLedgerStorage</strong>: additional 8 bytes per entry for entry location.</li><li><strong>Metadata</strong>: on zookeeper, we need a 128 bit ledger manager, that means more znode |
| hierarchy than 64 bit ledger manager. Etcd like key/value metadata store is probably |
| more preferrable for 128 bit ledger manager.</li></ul><p>However increasing ledger id from 64 bits to 128 bits can get rid of the only remaining |
| central point, since we don't need to use zookeeper for ledger id generation. The id |
| generation can become decentralized. </p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="proposed-changes">Proposed Changes<a href="#proposed-changes" class="hash-link" aria-label="Direct link to Proposed Changes" title="Direct link to Proposed Changes"></a></h3><p>All the required changes are described above. In summary, the changes can |
| happen in following 2 phases:</p><ol><li>Ensure all components have <code>ledgerScopeId</code> added (both wire protocol, storage and such). |
| Assuming <code>ledgerScopeId</code> will be 0. The changes can happen independently and ensure |
| they are backward compatible with old clients.</li><li>Add <code>ledgerScopeId</code> into public API, so application can start using <code>ledgerScopeId</code>. |
| After that, applications can use UUID to generate ledger id and break UUID into two parts, |
| one is <code>ledgerScopeId</code>, while the other one is <code>ledgerId</code>.</li></ol><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compatibility-deprecation-and-migration-plan">Compatibility, Deprecation, and Migration Plan<a href="#compatibility-deprecation-and-migration-plan" class="hash-link" aria-label="Direct link to Compatibility, Deprecation, and Migration Plan" title="Direct link to Compatibility, Deprecation, and Migration Plan"></a></h3><p>All the changes are backward compatible, since we are doing the changes by adding an optional |
| field <code>ledgerScopeId</code>. Old clients can still operating in the mode of <code>ledgerScopeId == 0</code>. |
| The new application can activate the feature by starting using <code>ledgerScopeId</code> in the new API.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test-plan">Test Plan<a href="#test-plan" class="hash-link" aria-label="Direct link to Test Plan" title="Direct link to Test Plan"></a></h3><ol><li>Add unit tests for individual components on introducing <code>ledgerScopeId</code>.</li><li>Add backward compatibility tests for individual components.</li><li>Add end-to-end integration tests for introducing <code>ledgerScopeId</code>.</li><li>Add end-to-end backward compatibility tests.</li></ol><h3 class="anchor anchorWithStickyNavbar_LWe7" id="rejected-alternatives">Rejected Alternatives<a href="#rejected-alternatives" class="hash-link" aria-label="Direct link to Rejected Alternatives" title="Direct link to Rejected Alternatives"></a></h3><p>N/A</p></article></div><div class="col col--2"><div class="tableOfContents_bqdL thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#motivation" class="table-of-contents__link toc-highlight">Motivation</a></li><li><a href="#128-bits" class="table-of-contents__link toc-highlight">128 bits</a></li><li><a href="#public-interfaces" class="table-of-contents__link toc-highlight">Public Interfaces</a></li><li><a href="#performance-concerns" class="table-of-contents__link toc-highlight">Performance Concerns</a></li><li><a href="#proposed-changes" class="table-of-contents__link toc-highlight">Proposed Changes</a></li><li><a href="#compatibility-deprecation-and-migration-plan" class="table-of-contents__link toc-highlight">Compatibility, Deprecation, and Migration Plan</a></li><li><a href="#test-plan" class="table-of-contents__link toc-highlight">Test Plan</a></li><li><a href="#rejected-alternatives" class="table-of-contents__link toc-highlight">Rejected Alternatives</a></li></ul></div></div></div></main></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Documentation</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/overview">Overview</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/getting-started/installation">Getting started</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/deployment/manual">Deployment</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/admin/bookies">Administration</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/api/overview">API</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/security/overview">Security</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/development/protocol">Development</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/reference/config">Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/community/mailing-lists">Mailing lists</a></li><li class="footer__item"><a class="footer__link-item" href="/community/slack">Slack</a></li><li class="footer__item"><a href="https://github.com/apache/bookkeeper" target="_blank" rel="noopener noreferrer" class="footer__link-item">Github<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://twitter.com/asfbookkeeper" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">Project</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/project/who">Who are we?</a></li><li class="footer__item"><a class="footer__link-item" href="/project/bylaws">Bylaws</a></li><li class="footer__item"><a href="https://apache.org/licenses" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/project/privacy">Privacy policy</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright"><footer class="footer"> |
| <div class="container"> |
| <div class="content has-text-centered"> |
| <p> |
| Copyright © 2016 - 2024 <a href="https://www.apache.org/">The Apache Software Foundation</a>,<br> licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, version 2.0</a>. |
| </p> |
| <p> |
| Apache BookKeeper, BookKeeper®, Apache®, the Apache feature logo, and the Apache BookKeeper logo are either registered trademarks or trademarks of The Apache Software Foundation. |
| </p> |
| </div> |
| </div> |
| </footer> |
| </div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.1c369ecb.js"></script> |
| <script src="/assets/js/main.812b2dbb.js"></script> |
| </body> |
| </html> |
