| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/bk-logo.svg" alt="Apache Bookkeeper" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/bk-logo.svg" alt="Apache Bookkeeper" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache BookKeeper</b></a><a class="navbar__item navbar__link" href="/docs/overview/">Documentation</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/community/mailing-lists">Mailing lists</a></li><li><a class="dropdown__link" href="/community/slack">Slack</a></li><li><a href="https://github.com/apache/bookkeeper/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Github issues<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/community/releases">Release management</a></li><li><a class="dropdown__link" href="/community/meeting">Community meetings</a></li><li><a class="dropdown__link" href="/community/contributing">Contribution guide</a></li><li><a class="dropdown__link" href="/community/coding-guide">Coding guide</a></li><li><a class="dropdown__link" href="/community/testing">Testing guide</a></li><li><a class="dropdown__link" href="/community/issue-report">Issue report guide</a></li><li><a class="dropdown__link" href="/community/release-guide">Release guide</a></li><li><a class="dropdown__link" href="/community/presentations">Presentations</a></li><li><a class="dropdown__link" href="/community/bookkeeper-proposals">BookKeeper proposals (BP)</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Project</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/project/who">Who are we?</a></li><li><a class="dropdown__link" href="/project/bylaws">Bylaws</a></li><li><a href="https://apache.org/licenses" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/project/privacy">Privacy policy</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/admin/autorecovery">4.17.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/overview/">Next</a></li><li><a class="dropdown__link" href="/docs/admin/autorecovery">4.17.0</a></li><li><a class="dropdown__link" href="/docs/4.16.5/overview/">4.16.5</a></li><li><a class="dropdown__link" href="/docs/4.15.5/overview/">4.15.5</a></li><li><a class="dropdown__link" href="/docs/4.14.8/overview/">4.14.8</a></li><li><a class="dropdown__link" href="/docs/4.13.0/overview/">4.13.0</a></li><li><a class="dropdown__link" href="/docs/4.12.1/overview/">4.12.1</a></li><li><a class="dropdown__link" href="/docs/4.11.1/overview/">4.11.1</a></li><li><a class="dropdown__link" href="/docs/4.10.0/overview/">4.10.0</a></li><li><a class="dropdown__link" href="/docs/4.9.2/overview/">4.9.2</a></li><li><a class="dropdown__link" href="/docs/4.8.2/overview/">4.8.2</a></li><li><a class="dropdown__link" href="/docs/4.7.3/overview/">4.7.3</a></li><li><a class="dropdown__link" href="/docs/4.6.2/overview/">4.6.2</a></li><li><a class="dropdown__link" href="/docs/4.5.1/overview/">4.5.1</a></li></ul></div><a class="navbar__item navbar__link" href="/releases">Download</a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><main class="container container--fluid margin-vert--lg"><div class="row mdxPageWrapper_j9I6"><div class="col col--8"><article><h1>BP-28: use etcd as metadata store</h1><h3 class="anchor anchorWithStickyNavbar_LWe7" id="motivation">Motivation<a href="#motivation" class="hash-link" aria-label="Direct link to Motivation" title="Direct link to Motivation"></a></h3><p>Currently bookkeeper uses zookeeper as the metadata store. However there is a couple of issues with current approach, especially using zookeeper.</p><p>These issues includes:</p><ol><li>You need to allocate special nodes for zookeeper. These nodes need to be treated specially, and have their own monitoring. |
| the scale goes up (e.g. the number of clients and watcher increase). The issues usually comes from <em>session expires</em> and <em>watcher</em>.</li></ol><ul><li><em>Session Expires</em>: For simplicity, ZooKeeper ties session state directly with connection state. So when a connection is broken, a session is usually expired (unless it reconnects before session expires), and when a session is expired, the underlying connection can not be used anymore, the application has to close the connection and re-establish a new client (a new connection). It is understandable that it makes zookeeper development easy. However in reality, it means if you can not establish a session, you can’t use this connection and you have to create new connections. Once your zookeeper cluster is in a bad state (e.g. network issue or jvm gc), the whole cluster is usually unable to recover because of the connection storm introduced by session expires.</li><li><em>Watchers</em>: The zookeeper watcher is one time watcher, applications can’t reliably use it to get updates. In order to set a watcher, you have to read a znode or get children. Imagine such a use case, clients are watching a list of znodes (e.g. list of bookies), when those clients expire, they have to get the list of znodes in order to rewatch the list, even the list is never changed.</li><li>The combination of session expires and watchers is often the root cause of critical zookeeper outages.</li></ul><p>This proposal is to explore other existing systems such as etcd as the metadata store. Using Etcd doesn't address concerns #1, however it might potentially |
| Both proposals are exploring a better metadata storage solution for bookkeeper.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="public-interfaces">Public Interfaces<a href="#public-interfaces" class="hash-link" aria-label="Direct link to Public Interfaces" title="Direct link to Public Interfaces"></a></h3><p>A new metadata store module <code>metadata-store-etcd</code> will be added to bookkeeper. This module will be implementing all the required metadata interfaces:</p><p>These interfaces include:</p><ul><li>RegistrationClient</li><li>RegistrationManager</li><li>LayoutManager</li><li>LedgerIdGenerator</li><li>LedgerManager</li><li>LedgerUnderreplicatedManager</li><li>LedgerManagerFactory</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="proposed-changes">Proposed Changes<a href="#proposed-changes" class="hash-link" aria-label="Direct link to Proposed Changes" title="Direct link to Proposed Changes"></a></h3><p>Since Etcd provides a key/value model rather than a tree-like structure. So the metadata will be organized in a key/value way as below:</p><ul><li><code>scope</code>: the prefix used for prefixing all the keys used for storing the metadata for a given cluster. The <code>scope</code> is effectively same as <code>zkLedgerRootPath</code>.</li><li><code>scope</code>/LAYOUT: key for storing layout data</li><li><code>scope</code>/INSTANCEID: key for storing instance id</li><li><code>scope</code>/IDGEN/<code><bucket></code>: key for id generation. <code><bucket></code> is to allow concurrent id generation.</li><li><code>scope</code>/cookies/<code><bookieid></code>: key for storing a bookie's cookie</li><li><code>scope</code>/available/readwrite/<code><bookieid></code>: key for registering readwrite bookie. (lease will be applied to this key)</li><li><code>scope</code>/available/readonly/<code><bookieid></code>: key for registering readonly bookie. (lease will be applied to this key)</li><li><code>scope</code>/ledgers/<code><ledgerid></code>: key for storing a ledger metadata. <code><ledgerid></code> is <code>String.format("%19d", ledgerId)</code>, pre-padding with 0s to make sure ledgers are stored in order.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="registration-manager">Registration Manager<a href="#registration-manager" class="hash-link" aria-label="Direct link to Registration Manager" title="Direct link to Registration Manager"></a></h4><ul><li>Bookie Register: write to key "<code>scope</code>/available/readwrite/<code><bookieid></code>" with a keepalive lease.</li><li>Bookie Unregister: delete key "<code>scope</code>/available/readwrite/<code><bookieid></code>".</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="registration-client">Registration Client<a href="#registration-client" class="hash-link" aria-label="Direct link to Registration Client" title="Direct link to Registration Client"></a></h4><ul><li>Get readwrite bookies: range operation to fetch keys between "<code>scope</code>/available/readwrite/" and "<code>scope</code>/available/readwrite_end/".</li><li>Get readonly bookies: range operation to fetch keys between "<code>scope</code>/available/readonly/" and "<code>scope</code>/available/readonly_end/".</li><li>Watch readwrite bookies: watch operation to watch keys between "<code>scope</code>/available/readwrite/" and "<code>scope</code>/available/readwrite_end/".</li><li>Watch readonly bookies: watch operation to watch keys between "<code>scope</code>/available/readonly/" and "<code>scope</code>/available/readonly_end/". </li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="layout-manager">Layout Manager<a href="#layout-manager" class="hash-link" aria-label="Direct link to Layout Manager" title="Direct link to Layout Manager"></a></h4><ul><li>Create layout: a txn operation: put layout data to key "<code>scope</code>/LAYOUT" when this key doesn't exist.</li><li>Delete layout: a delete operation on "<code>scope</code>/LAYOUT"</li><li>Read layout: a get operation on "<code>scope</code>/LAYOUT"</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ledger-id-generator">Ledger Id Generator<a href="#ledger-id-generator" class="hash-link" aria-label="Direct link to Ledger Id Generator" title="Direct link to Ledger Id Generator"></a></h4><ul><li>Id generation: get key, increment by 1 and then update with a txn operation.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="ledger-manager">Ledger Manager<a href="#ledger-manager" class="hash-link" aria-label="Direct link to Ledger Manager" title="Direct link to Ledger Manager"></a></h4><ul><li>Create Ledger: a txn operation to put ledger metadata to key "<code>scope</code>/ledgers/<code><ledgerid></code>" when this key doesn't exist</li><li>Write Ledger: a txn operation to put ledger metadata to key "<code>scope</code>/ledgers/<code><ledgerid></code>" when key version matches</li><li>Delete Ledger: a delete operation on key "<code>scope</code>/ledgers/<code><ledgerid></code>"</li><li>Read Ledger: a get operation on key "<code>scope</code>/ledgers/<code><ledgerid></code>"</li><li>Iteration: a range operation fetch keys between "<code>scope</code>/ledgers/" and "<code>scope</code>/ledgers_end/"</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compatibility-deprecation-and-migration-plan">Compatibility, Deprecation, and Migration Plan<a href="#compatibility-deprecation-and-migration-plan" class="hash-link" aria-label="Direct link to Compatibility, Deprecation, and Migration Plan" title="Direct link to Compatibility, Deprecation, and Migration Plan"></a></h3><ul><li>This is a new metadata store. No impacts to existing users.</li><li>We are not planning to implement any migration tools any time soon. This only for new users or setting up new clusters.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test-plan">Test Plan<a href="#test-plan" class="hash-link" aria-label="Direct link to Test Plan" title="Direct link to Test Plan"></a></h3><ul><li>Unit tests for all the metadata manager implementation.</li><li>End-to-end integration tests</li><li>Jepsen tests</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="rejected-alternatives">Rejected Alternatives<a href="#rejected-alternatives" class="hash-link" aria-label="Direct link to Rejected Alternatives" title="Direct link to Rejected Alternatives"></a></h3><p>N/A</p></article></div><div class="col col--2"><div class="tableOfContents_bqdL thin-scrollbar"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#motivation" class="table-of-contents__link toc-highlight">Motivation</a></li><li><a href="#public-interfaces" class="table-of-contents__link toc-highlight">Public Interfaces</a></li><li><a href="#proposed-changes" class="table-of-contents__link toc-highlight">Proposed Changes</a></li><li><a href="#compatibility-deprecation-and-migration-plan" class="table-of-contents__link toc-highlight">Compatibility, Deprecation, and Migration Plan</a></li><li><a href="#test-plan" class="table-of-contents__link toc-highlight">Test Plan</a></li><li><a href="#rejected-alternatives" class="table-of-contents__link toc-highlight">Rejected Alternatives</a></li></ul></div></div></div></main></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Documentation</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/overview">Overview</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/getting-started/installation">Getting started</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/deployment/manual">Deployment</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/admin/bookies">Administration</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/api/overview">API</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/security/overview">Security</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/development/protocol">Development</a></li><li class="footer__item"><a class="footer__link-item" href="/docs/reference/config">Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/community/mailing-lists">Mailing lists</a></li><li class="footer__item"><a class="footer__link-item" href="/community/slack">Slack</a></li><li class="footer__item"><a href="https://github.com/apache/bookkeeper" target="_blank" rel="noopener noreferrer" class="footer__link-item">Github<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://twitter.com/asfbookkeeper" target="_blank" rel="noopener noreferrer" class="footer__link-item">Twitter<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><div class="col footer__col"><div class="footer__title">Project</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/project/who">Who are we?</a></li><li class="footer__item"><a class="footer__link-item" href="/project/bylaws">Bylaws</a></li><li class="footer__item"><a href="https://apache.org/licenses" target="_blank" rel="noopener noreferrer" class="footer__link-item">License<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a class="footer__link-item" href="/project/privacy">Privacy policy</a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Sponsorship<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Thanks<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright"><footer class="footer"> |
