| <!DOCTYPE html> |
| <html> |
| <head> |
| <title>Apache BookKeeper™ - Apache BookKeeper 4.5.0 Release Notes</title> |
| |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| |
| <link rel="stylesheet" href="/css/normalize.css"> |
| <link rel="stylesheet" href="/css/tippy.css"> |
| <link rel="stylesheet" href="/css/style.css"> |
| |
| <link rel="shortcut icon" href="/img/favicon.ico"> |
| |
| <script src="/js/tippy.min.js"></script> |
| |
| <script type="text/javascript"> |
| var shiftWindow = function() { scrollBy(0, -25); }; |
| window.addEventListener("hashchange", shiftWindow); |
| window.addEventListener("pageshow", shiftWindow); |
| function load() { if (window.location.hash) shiftWindow(); } |
| </script> |
| </head> |
| <body class="body"> |
| <main class="main"> |
| |
| <nav class="navbar bk-topnav"> |
| <div class="navbar-brand"> |
| <a class="navbar-item bk-brand" href="/"> |
| Apache BookKeeper™ |
| </a> |
| |
| <div class="navbar-burger burger" data-target="bkNav"> |
| <span></span> |
| <span></span> |
| <span></span> |
| </div> |
| </div> |
| |
| <div id="bkNav" class="navbar-menu"> |
| <div class="navbar-start"> |
| <div class="navbar-item has-dropdown is-hoverable"> |
| <a class="navbar-link">Documentation</a> |
| <div class="navbar-dropdown is-boxed"> |
| <a class="navbar-item" href="/docs/latest/overview/overview"> |
| Version 4.14.0-SNAPSHOT |
| <span class="tag is-warning">Development</span> |
| </a> |
| <a class="navbar-item" href="/docs/latest/api/javadoc"> |
| <span class="icon bk-javadoc-icon"> |
| <img src="/img/java-icon.svg"> |
| </span> |
| Javadoc |
| </a> |
| <hr class="dropdown-divider"> |
| |
| <a class="navbar-item" href="/docs/4.13.0/overview/overview"> |
| Release 4.13.0 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.12.1/overview/overview"> |
| Release 4.12.1 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.12.0/overview/overview"> |
| Release 4.12.0 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.11.1/overview/overview"> |
| Release 4.11.1 |
| |
| <span class="tag is-success">Stable</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.11.0/overview/overview"> |
| Release 4.11.0 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.10.0/overview/overview"> |
| Release 4.10.0 |
| |
| </a> |
| |
| |
| <a class="navbar-item" href="/archives/docs/r4.9.2"> |
| Release 4.9.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.9.1"> |
| Release 4.9.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.9.0"> |
| Release 4.9.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.8.2"> |
| Release 4.8.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.8.1"> |
| Release 4.8.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.8.0"> |
| Release 4.8.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.3"> |
| Release 4.7.3 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.2"> |
| Release 4.7.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.1"> |
| Release 4.7.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.0"> |
| Release 4.7.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.6.2"> |
| Release 4.6.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.6.1"> |
| Release 4.6.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.6.0"> |
| Release 4.6.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.5.1"> |
| Release 4.5.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.5.0"> |
| Release 4.5.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.4.0"> |
| Release 4.4.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.3.2"> |
| Release 4.3.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.3.1"> |
| Release 4.3.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.3.0"> |
| Release 4.3.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.4"> |
| Release 4.2.4 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.3"> |
| Release 4.2.3 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.2"> |
| Release 4.2.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.1"> |
| Release 4.2.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.0"> |
| Release 4.2.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.1.0"> |
| Release 4.1.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.0.0"> |
| Release 4.0.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| </div> |
| </div> |
| |
| <div class="navbar-item has-dropdown is-hoverable"> |
| <a class="navbar-link">Community</a> |
| <div class="navbar-dropdown is-boxed"> |
| <a class="navbar-item" href="/community/mailing-lists">Mailing lists</a> |
| <a class="navbar-item" href="/community/slack">Slack</a> |
| <a class="navbar-item" href="https://github.com/apache/bookkeeper/issues">Github Issues</a> |
| <a class="navbar-item" href="/community/releases">Release Management</a> |
| <a class="navbar-item" href="/community/meeting">Community Meetings</a> |
| <hr class="dropdown-divider"> |
| <a class="navbar-item" href="/community/contributing">Contribution Guide</a> |
| <a class="navbar-item" href="/community/coding_guide">Coding Guide</a> |
| <a class="navbar-item" href="/community/testing">Testing Guide</a> |
| <a class="navbar-item" href="/community/issue-report">Issue Report Guide</a> |
| <a class="navbar-item" href="/community/release_guide">Release Guide</a> |
| <hr class="dropdown-divider"> |
| <a class="navbar-item" href="/community/presentations">Presentations</a> |
| <a class="navbar-item" href="/community/bookkeeper_proposals">BookKeeper Proposals</a> |
| </div> |
| </div> |
| |
| <div class="navbar-item has-dropdown is-hoverable"> |
| <a class="navbar-link">Project</a> |
| <div class="navbar-dropdown is-boxed"> |
| <a class="navbar-item" href="/project/who">Who are we?</a> |
| <a class="navbar-item" href="/project/bylaws">Bylaws</a> |
| <a class="navbar-item" href="http://www.apache.org/licenses/">License</a> |
| <hr class="dropdown-divider"> |
| <a class="navbar-item" href="/project/privacy">Privacy policy</a> |
| <a class="navbar-item" href="http://www.apache.org/foundation/sponsorship.html">Sponsorship</a> |
| <a class="navbar-item" href="http://www.apache.org/foundation/thanks.html">Thanks</a> |
| </div> |
| </div> |
| </div> |
| |
| <div class="navbar-end"> |
| <div class="navbar-item"> |
| <div class="field is-grouped"> |
| <p class="control"> |
| <a class="button bk-twitter" href="https://twitter.com/asfbookkeeper"> |
| <span class="icon"> |
| <i class="fa fa-twitter"></i> |
| </span> |
| <span>Twitter</span> |
| </a> |
| </p> |
| <p class="control"> |
| <a class="button" href="https://github.com/apache/bookkeeper"> |
| <span class="icon"> |
| <i class="fa fa-github"></i> |
| </span> |
| <span>GitHub</span> |
| </a> |
| </p> |
| <p class="control"> |
| <a class="button is-primary" href="/releases"> |
| <span class="icon"> |
| <i class="fa fa-download"></i> |
| </span> |
| <span>Download</span> |
| </a> |
| </p> |
| </div> |
| </div> |
| </div> |
| </div> |
| </nav> |
| |
| |
| <div class="bk-docs-container"> |
| <div class="columns is-gapless"> |
| <div class="column is-2 is-hidden-mobile"> |
| <div class="container"> |
| |
| <aside class="sidebar"> |
| |
| <a class="button is-info"> |
| Version: 4.5.0 |
| </a> |
| <hr /> |
| |
| <p> |
| Getting started |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../getting-started/installation"> |
| Installation |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../getting-started/run-locally"> |
| Run bookies locally |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../getting-started/concepts"> |
| Concepts and architecture |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Deployment |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../deployment/manual"> |
| Manual deployment |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../deployment/dcos"> |
| BookKeeper on DC/OS |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../deployment/kubernetes"> |
| BookKeeper on Kubernetes |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Administration |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../admin/bookies"> |
| BookKeeper administration |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/autorecovery"> |
| AutoRecovery |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/metrics"> |
| Metric collection |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/upgrade"> |
| Upgrade |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/http"> |
| BookKeeper Admin REST API |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/decomission"> |
| Decommissioning Bookies |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| API |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../api/overview"> |
| Overview |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/ledger-api"> |
| Ledger API |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/ledger-adv-api"> |
| Advanced Ledger API |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/distributedlog-api"> |
| DistributedLog |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/javadoc"> |
| Java API Docs |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Security |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../security/overview"> |
| Overview |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../security/tls"> |
| TLS Authentication |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../security/sasl"> |
| SASL Authentication |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../security/zookeeper"> |
| ZooKeeper Authentication |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Development |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../development/protocol"> |
| BookKeeper protocol |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Reference |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../reference/config"> |
| Configuration |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../reference/cli"> |
| Command-line tools |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../reference/metrics"> |
| Metrics |
| </a> |
| </li> |
| |
| </ul> |
| |
| </aside> |
| |
| |
| </div> |
| </div> |
| |
| <div class="column is-8 bk-docs-block"> |
| <header class="docs-title"> |
| <nav class="level bk-level"> |
| <div class="level-left"> |
| <div class="level-item"> |
| <h1 class="title">Apache BookKeeper 4.5.0 Release Notes</h1> |
| </div> |
| </div> |
| |
| </nav> |
| |
| |
| </header> |
| |
| <hr /> |
| |
| <div class="content"> |
| <section class="bk-main-content"> |
| <p>This is the fifth release of BookKeeper as an Apache Top Level Project!</p> |
| |
| <p>The 4.5.0 release incorporates hundreds of new fixes, improvements, and features since previous major release, 4.4.0, |
| which was released over a year ago. It is a big milestone in Apache BookKeeper community, converging from three |
| main branches (Salesforce, Twitter and Yahoo).</p> |
| |
| <p>Apache BookKeeper users are encouraged to upgrade to 4.5.0. The technical details of this release are summarized |
| below.</p> |
| |
| <h2 id="highlights">Highlights</h2> |
| |
| <p>The main features in 4.5.0 cover are around following areas:</p> |
| |
| <ul> |
| <li>Dependencies Upgrade</li> |
| <li>Security</li> |
| <li>Public API</li> |
| <li>Performance</li> |
| <li>Operations</li> |
| </ul> |
| |
| <h3 id="dependencies-upgrade">Dependencies Upgrade</h3> |
| |
| <p>Here is a list of dependencies upgraded in 4.5.0:</p> |
| |
| <ul> |
| <li>Moved the developement from Java 7 to Java 8.</li> |
| <li>Upgrade Protobuf to <code class="highlighter-rouge">2.6</code>.</li> |
| <li>Upgrade ZooKeeper from <code class="highlighter-rouge">3.4</code> to <code class="highlighter-rouge">3.5</code>.</li> |
| <li>Upgrade Netty to <code class="highlighter-rouge">4.1</code>.</li> |
| <li>Upgrade Guava to <code class="highlighter-rouge">20.0</code>.</li> |
| <li>Upgrade SLF4J to <code class="highlighter-rouge">1.7.25</code>.</li> |
| <li>Upgrade Codahale to <code class="highlighter-rouge">3.1.0</code>.</li> |
| </ul> |
| |
| <h3 id="security">Security</h3> |
| |
| <p>Prior to this release, Apache BookKeeper only supports simple <code class="highlighter-rouge">DIGEST-MD5</code> type authentication.</p> |
| |
| <p>With this release of Apache BookKeeper, a number of feature are introduced that can be used, together of separately, |
| to secure a BookKeeper cluster.</p> |
| |
| <p>The following security features are currently supported.</p> |
| |
| <ul> |
| <li>Authentication of connections to bookies from clients, using either <code class="highlighter-rouge">TLS</code> or `SASL (Kerberos).</li> |
| <li>Authentication of connections from clients, bookies, autorecovery daemons to <code class="highlighter-rouge">ZooKeeper</code>, when using zookeeper |
| based ledger managers.</li> |
| <li>Encryption of data transferred between bookies and clients, between bookies and autorecovery daemons using <code class="highlighter-rouge">TLS</code>.</li> |
| </ul> |
| |
| <p>It’s worth noting that those security features are optional - non-secured clusters are supported, as well as a mix |
| of authenticated, unauthenticated, encrypted and non-encrypted clients.</p> |
| |
| <p>For more details, have a look at <a href="../../security/overview">BookKeeper Security</a>.</p> |
| |
| <h3 id="public-api">Public API</h3> |
| |
| <p>There are multiple new client features introduced in 4.5.0.</p> |
| |
| <h4 id="ledgerhandleadv">LedgerHandleAdv</h4> |
| |
| <p>The [Ledger API] is the low level API provides by BookKeeper for interacting with <code class="highlighter-rouge">ledgers</code> in a bookkeeper cluster. |
| It is simple but not flexible on ledger id or entry id generation. Apache BookKeeper introduces <code class="highlighter-rouge">LedgerHandleAdv</code> |
| as an extension of existing <code class="highlighter-rouge">LedgerHandle</code> for advanced usage. The new <code class="highlighter-rouge">LedgerHandleAdv</code> allows applications providing |
| its own <code class="highlighter-rouge">ledger-id</code> and assigning <code class="highlighter-rouge">entry-id</code> on adding entries.</p> |
| |
| <p>See <a href="../../api/ledger-adv-api">Ledger Advanced API</a> for more details.</p> |
| |
| <h4 id="long-poll">Long Poll</h4> |
| |
| <p><code class="highlighter-rouge">Long Poll</code> is a main feature that <a href="https://distributedlog.io">DistributedLog</a> uses to achieve low-latency tailing. |
| This big feature has been merged back in 4.5.0 and available to BookKeeper users.</p> |
| |
| <p>This feature includes two main changes, one is <code class="highlighter-rouge">LastAddConfirmed</code> piggyback, while the other one is a new <code class="highlighter-rouge">long poll</code> read API.</p> |
| |
| <p>The first change piggyback the latest <code class="highlighter-rouge">LastAddConfirm</code> along with the read response, so your <code class="highlighter-rouge">LastAddConfirmed</code> will be automatically advanced |
| when your read traffic continues. It significantly reduces the traffic to explicitly polling <code class="highlighter-rouge">LastAddConfirmed</code> and hence reduces the end-to-end latency.</p> |
| |
| <p>The second change provides a new <code class="highlighter-rouge">long poll</code> read API, allowing tailing-reads without polling <code class="highlighter-rouge">LastAddConfirmed</code> everytime after readers exhaust known entries. |
| Although <code class="highlighter-rouge">long poll</code> API brings great latency improvements on tailing reads, it is still a very low-level primitive. |
| It is still recommended to use high level API (e.g. <a href="../../api/distributedlog-api">DistributedLog API</a>) for tailing and streaming use cases.</p> |
| |
| <p>See <a href="https://bookkeeper.apache.org/distributedlog/docs/latest/user_guide/design/main.html#streaming-reads">Streaming Reads</a> for more details.</p> |
| |
| <h4 id="explicit-lac">Explicit LAC</h4> |
| |
| <p>Prior to 4.5.0, the <code class="highlighter-rouge">LAC</code> is only advanced when subsequent entries are added. If there is no subsequent entries added, |
| the last entry written will not be visible to readers until the ledger is closed. High-level client (e.g. DistributedLog) or applications |
| has to work around this by writing some sort of <code class="highlighter-rouge">control records</code> to advance <code class="highlighter-rouge">LAC</code>.</p> |
| |
| <p>In 4.5.0, a new <code class="highlighter-rouge">explicit lac</code> feature is introduced to periodically advance <code class="highlighter-rouge">LAC</code> if there are not subsequent entries added. This feature |
| can be enabled by setting <code class="highlighter-rouge">explicitLacInterval</code> to a positive value.</p> |
| |
| <h3 id="performance">Performance</h3> |
| |
| <p>There are a lot for performance related bug fixes and improvements in 4.5.0. These changes includes:</p> |
| |
| <ul> |
| <li>Upgraded netty from 3.x to 4.x to leverage buffer pooling and reduce memory copies.</li> |
| <li>Moved developement from Java 7 to Java 8 to take advantage of Java 8 features.</li> |
| <li>A lot of improvements around scheduling and threading on <code class="highlighter-rouge">bookies</code>.</li> |
| <li>Delay ensemble change to improve tail latency.</li> |
| <li>Parallel ledger recovery to improve the recovery speed.</li> |
| <li>…</li> |
| </ul> |
| |
| <p>We outlined following four changes as below. For a complete list of performance improvements, please checkout the <code class="highlighter-rouge">full list of changes</code> at the end.</p> |
| |
| <h4 id="netty-4-upgrade">Netty 4 Upgrade</h4> |
| |
| <p>The major performance improvement introduced in 4.5.0, is upgrading netty from 3.x to <a href="http://netty.io/wiki/new-and-noteworthy-in-4.0.html">4.x</a>.</p> |
| |
| <p>For more details, please read <a href="../../admin/upgrade">upgrade guide</a> about the netty related tips when upgrading bookkeeper from 4.4.0 to 4.5.0.</p> |
| |
| <h4 id="delay-ensemble-change">Delay Ensemble Change</h4> |
| |
| <p><code class="highlighter-rouge">Ensemble Change</code> is a feature that Apache BookKeeper uses to achieve high availability. However it is an expensive metadata operation. |
| Especially when Apache BookKeeper is deployed in a multiple data-centers environment, losing a data center will cause churn of metadata |
| operations due to ensemble changes. <code class="highlighter-rouge">Delay Ensemble Change</code> is introduced in 4.5.0 to overcome this problem. Enabling this feature means |
| an <code class="highlighter-rouge">Ensemble Change</code> will only occur when clients can’t receive enough valid responses to satisfy <code class="highlighter-rouge">ack-quorum</code> constraint. This feature |
| improves the tail latency.</p> |
| |
| <p>To enable this feature, please set <code class="highlighter-rouge">delayEnsembleChange</code> to <code class="highlighter-rouge">true</code> on your clients.</p> |
| |
| <h4 id="parallel-ledger-recovery">Parallel Ledger Recovery</h4> |
| |
| <p>BookKeeper clients recovers entries one-by-one during ledger recovery. If a ledger has very large volumn of traffic, it will have |
| large number of entries to recover when client failures occur. BookKeeper introduces <code class="highlighter-rouge">parallel ledger recovery</code> in 4.5.0 to allow |
| batch recovery to improve ledger recovery speed.</p> |
| |
| <p>To enable this feature, please set <code class="highlighter-rouge">enableParallelRecoveryRead</code> to <code class="highlighter-rouge">true</code> on your clients. You can also set <code class="highlighter-rouge">recoveryReadBatchSize</code> |
| to control the batch size of recovery read.</p> |
| |
| <h4 id="multiple-journals">Multiple Journals</h4> |
| |
| <p>Prior to 4.5.0, bookies are only allowed to configure one journal device. If you want to have high write bandwidth, you can raid multiple |
| disks into one device and mount that device for jouranl directory. However because there is only one journal thread, this approach doesn’t |
| actually improve the write bandwidth.</p> |
| |
| <p>BookKeeper introduces multiple journal directories support in 4.5.0. Users can configure multiple devices for journal directories.</p> |
| |
| <p>To enable this feature, please use <code class="highlighter-rouge">journalDirectories</code> rather than <code class="highlighter-rouge">journalDirectory</code>.</p> |
| |
| <h3 id="operations">Operations</h3> |
| |
| <h4 id="longhierarchicalledgermanager">LongHierarchicalLedgerManager</h4> |
| |
| <p>Apache BookKeeper supports pluggable metadata store. By default, it uses Apache ZooKeeper as its metadata store. Among the zookeeper-based |
| ledger manager implementations, <code class="highlighter-rouge">HierarchicalLedgerManager</code> is the most popular and widely adopted ledger manager. However it has a major |
| limitation, which it assumes <code class="highlighter-rouge">ledger-id</code> is a 32-bits integer. It limits the number of ledgers to <code class="highlighter-rouge">2^32</code>.</p> |
| |
| <p><code class="highlighter-rouge">LongHierarchicalLedgerManager</code> is introduced to overcome this limitation.</p> |
| |
| <p>See <a href="../../getting-started/concepts/#ledger-manager">Ledger Manager</a> for more details.</p> |
| |
| <h4 id="weight-based-placement-policy">Weight-based placement policy</h4> |
| |
| <p><code class="highlighter-rouge">Rack-Aware</code> and <code class="highlighter-rouge">Region-Aware</code> placement polices are the two available placement policies in BookKeeper client. It places ensembles based |
| on users’ configured network topology. However they both assume that all nodes are equal. <code class="highlighter-rouge">weight-based</code> placement is introduced in 4.5.0 to |
| improve the existing placement polices. <code class="highlighter-rouge">weight-based</code> placement was not built as separated polices. It is built in the existing placement policies. |
| If you are using <code class="highlighter-rouge">Rack-Aware</code> or <code class="highlighter-rouge">Region-Aware</code>, you can simply enable <code class="highlighter-rouge">weight-based</code> placement by setting <code class="highlighter-rouge">diskWeightBasedPlacementEnabled</code> to <code class="highlighter-rouge">true</code>.</p> |
| |
| <h4 id="customized-ledger-metadata">Customized Ledger Metadata</h4> |
| |
| <p>A <code class="highlighter-rouge">Map<String, byte[]></code> is introduced in ledger metadata in 4.5.0. Clients now are allowed to pass in a key/value map when creating ledgers. |
| This customized ledger metadata can be later on used by user defined placement policy. This extends the flexibility of bookkeeper API.</p> |
| |
| <h4 id="add-prometheus-stats-provider">Add Prometheus stats provider</h4> |
| |
| <p>A new <a href="https://prometheus.io/">Prometheus</a> <a href="https://github.com/apache/bookkeeper/tree/master/bookkeeper-stats-providers/prometheus-metrics-provider">stats provider</a> |
| is introduce in 4.5.0. It simplies the metric collection when running bookkeeper on <a href="https://kubernetes.io/">kubernetes</a>.</p> |
| |
| <h4 id="add-more-tools-in-bookieshell">Add more tools in BookieShell</h4> |
| |
| <p><code class="highlighter-rouge">BookieShell</code> is the tool provided by Apache BooKeeper to operate clusters. There are multiple importants tools introduced in 4.5.0, for example, <code class="highlighter-rouge">decommissionbookie</code>, |
| <code class="highlighter-rouge">expandstorage</code>, <code class="highlighter-rouge">lostbookierecoverydelay</code>, <code class="highlighter-rouge">triggeraudit</code>.</p> |
| |
| <p>For the complete list of commands in <code class="highlighter-rouge">BookieShell</code>, please read <a href="../../reference/cli">BookKeeper CLI tool reference</a>.</p> |
| |
| <h2 id="full-list-of-changes">Full list of changes</h2> |
| |
| <h3 id="jira">JIRA</h3> |
| |
| <h4 id="sub-task">Sub-task</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-552">BOOKKEEPER-552</a>] - 64 Bits Ledger ID Generation |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-553">BOOKKEEPER-553</a>] - New LedgerManager for 64 Bits Ledger ID Management in ZooKeeper |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-588">BOOKKEEPER-588</a>] - SSL support |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-873">BOOKKEEPER-873</a>] - Enhance CreatedLedger API to accept ledgerId as input |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-949">BOOKKEEPER-949</a>] - Allow entryLog creation even when bookie is in RO mode for compaction |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-965">BOOKKEEPER-965</a>] - Long Poll: Changes to the Write Path |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-997">BOOKKEEPER-997</a>] - Wire protocol change for supporting long poll |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1017">BOOKKEEPER-1017</a>] - Create documentation for ZooKeeper ACLs |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1086">BOOKKEEPER-1086</a>] - Ledger Recovery - Refactor PendingReadOp |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1087">BOOKKEEPER-1087</a>] - Ledger Recovery - Add a parallel reading request in PendingReadOp |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1088">BOOKKEEPER-1088</a>] - Ledger Recovery - Add a ReadEntryListener to callback on individual request |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1089">BOOKKEEPER-1089</a>] - Ledger Recovery - allow batch reads in ledger recovery |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1092">BOOKKEEPER-1092</a>] - Ledger Recovery - Add Test Case for Parallel Ledger Recovery |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1093">BOOKKEEPER-1093</a>] - Piggyback LAC on ReadResponse |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1094">BOOKKEEPER-1094</a>] - Long Poll - Server and Client Side Changes |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1095">BOOKKEEPER-1095</a>] - Long Poll - Client side changes |
| </li> |
| </ul> |
| |
| <h4 id="bug">Bug</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-852">BOOKKEEPER-852</a>] - Release LedgerDescriptor and master-key objects when not used anymore |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-903">BOOKKEEPER-903</a>] - MetaFormat BookieShell Command is not deleting UnderReplicatedLedgers list from the ZooKeeper |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-907">BOOKKEEPER-907</a>] - for ReadLedgerEntriesCmd, EntryFormatter should be configurable and HexDumpEntryFormatter should be one of them |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-908">BOOKKEEPER-908</a>] - Case to handle BKLedgerExistException |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-924">BOOKKEEPER-924</a>] - addEntry() is susceptible to spurious wakeups |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-927">BOOKKEEPER-927</a>] - Extend BOOKKEEPER-886 to LedgerHandleAdv too (BOOKKEEPER-886: Allow to disable ledgers operation throttling) |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-933">BOOKKEEPER-933</a>] - ClientConfiguration always inherits System properties |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-938">BOOKKEEPER-938</a>] - LedgerOpenOp should use digestType from metadata |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-939">BOOKKEEPER-939</a>] - Fix typo in bk-merge-pr.py |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-940">BOOKKEEPER-940</a>] - Fix findbugs warnings after bumping to java 8 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-952">BOOKKEEPER-952</a>] - Fix RegionAwarePlacementPolicy |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-955">BOOKKEEPER-955</a>] - in BookKeeperAdmin listLedgers method currentRange variable is not getting updated to next iterator when it has run out of elements |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-956">BOOKKEEPER-956</a>] - HierarchicalLedgerManager doesn't work for ledgerid of length 9 and 10 because of order issue in HierarchicalLedgerRangeIterator |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-958">BOOKKEEPER-958</a>] - ZeroBuffer readOnlyBuffer returns ByteBuffer with 0 remaining bytes for length > 64k |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-959">BOOKKEEPER-959</a>] - ClientAuthProvider and BookieAuthProvider Public API used Protobuf Shaded classes |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-976">BOOKKEEPER-976</a>] - Fix license headers with "Copyright 2016 The Apache Software Foundation" |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-980">BOOKKEEPER-980</a>] - BookKeeper Tools doesn't process the argument correctly |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-981">BOOKKEEPER-981</a>] - NullPointerException in RackawareEnsemblePlacementPolicy while running in Docker Container |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-984">BOOKKEEPER-984</a>] - BookieClientTest.testWriteGaps tested |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-986">BOOKKEEPER-986</a>] - Handle Memtable flush failure |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-987">BOOKKEEPER-987</a>] - BookKeeper build is broken due to the shade plugin for commit ecbb053e6e |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-988">BOOKKEEPER-988</a>] - Missing license headers |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-989">BOOKKEEPER-989</a>] - Enable travis CI for bookkeeper git |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-999">BOOKKEEPER-999</a>] - BookKeeper client can leak threads |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1013">BOOKKEEPER-1013</a>] - Fix findbugs errors on latest master |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1018">BOOKKEEPER-1018</a>] - Allow client to select older V2 protocol (no protobuf) |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1020">BOOKKEEPER-1020</a>] - Fix Explicit LAC tests on master |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1021">BOOKKEEPER-1021</a>] - Improve the merge script to handle github reviews api |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1031">BOOKKEEPER-1031</a>] - ReplicationWorker.rereplicate fails to call close() on ReadOnlyLedgerHandle |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1044">BOOKKEEPER-1044</a>] - Entrylogger is not readding rolled logs back to the logChannelsToFlush list when exception happens while trying to flush rolled logs |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1047">BOOKKEEPER-1047</a>] - Add missing error code in ZK setData return path |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1058">BOOKKEEPER-1058</a>] - Ignore already deleted ledger on replication audit |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1061">BOOKKEEPER-1061</a>] - BookieWatcher should not do ZK blocking operations from ZK async callback thread |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1065">BOOKKEEPER-1065</a>] - OrderedSafeExecutor should only have 1 thread per bucket |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1071">BOOKKEEPER-1071</a>] - BookieRecoveryTest is failing due to a Netty4 IllegalReferenceCountException |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1072">BOOKKEEPER-1072</a>] - CompactionTest is flaky when disks are almost full |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1073">BOOKKEEPER-1073</a>] - Several stats provider related changes. |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1074">BOOKKEEPER-1074</a>] - Remove JMX Bean |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1075">BOOKKEEPER-1075</a>] - BK LedgerMetadata: more memory-efficient parsing of configs |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1076">BOOKKEEPER-1076</a>] - BookieShell should be able to read the 'FENCE' entry in the log |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1077">BOOKKEEPER-1077</a>] - BookKeeper: Local Bookie Journal and ledger paths |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1079">BOOKKEEPER-1079</a>] - shell lastMark throws NPE |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1098">BOOKKEEPER-1098</a>] - ZkUnderreplicationManager can build up an unbounded number of watchers |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1101">BOOKKEEPER-1101</a>] - BookKeeper website menus not working under https |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1102">BOOKKEEPER-1102</a>] - org.apache.bookkeeper.client.BookKeeperDiskSpaceWeightedLedgerPlacementTest.testDiskSpaceWeightedBookieSelectionWithBookiesBeingAdded is unreliable |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1103">BOOKKEEPER-1103</a>] - LedgerMetadataCreateTest bug in ledger id generation causes intermittent hang |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1104">BOOKKEEPER-1104</a>] - BookieInitializationTest.testWithDiskFullAndAbilityToCreateNewIndexFile testcase is unreliable |
| </li> |
| </ul> |
| |
| <h4 id="improvement">Improvement</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-612">BOOKKEEPER-612</a>] - RegionAwarePlacement Policy |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-748">BOOKKEEPER-748</a>] - Move fence requests out of read threads |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-757">BOOKKEEPER-757</a>] - Ledger Recovery Improvement |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-759">BOOKKEEPER-759</a>] - bookkeeper: delay ensemble change if it doesn't break ack quorum requirement |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-772">BOOKKEEPER-772</a>] - Reorder read sequnce |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-874">BOOKKEEPER-874</a>] - Explict LAC from Writer to Bookies |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-881">BOOKKEEPER-881</a>] - upgrade surefire plugin to 2.19 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-887">BOOKKEEPER-887</a>] - Allow to use multiple bookie journals |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-922">BOOKKEEPER-922</a>] - Create a generic (K,V) map to store ledger metadata |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-935">BOOKKEEPER-935</a>] - Publish sources and javadocs to Maven Central |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-937">BOOKKEEPER-937</a>] - Upgrade protobuf to 2.6 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-944">BOOKKEEPER-944</a>] - Multiple issues and improvements to BK Compaction. |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-945">BOOKKEEPER-945</a>] - Add counters to track the activity of auditor and replication workers |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-946">BOOKKEEPER-946</a>] - Provide an option to delay auto recovery of lost bookies |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-961">BOOKKEEPER-961</a>] - Assing read/write request for same ledger to a single thread |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-962">BOOKKEEPER-962</a>] - Add more journal timing stats |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-963">BOOKKEEPER-963</a>] - Allow to use multiple journals in bookie |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-964">BOOKKEEPER-964</a>] - Add concurrent maps and sets for primitive types |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-966">BOOKKEEPER-966</a>] - change the bookieServer cmdline to make conf-file and option co-exist |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-968">BOOKKEEPER-968</a>] - Entry log flushes happen on log rotation and cause long spikes in IO utilization |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-970">BOOKKEEPER-970</a>] - Bump zookeeper version to 3.5 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-971">BOOKKEEPER-971</a>] - update bk codahale stats provider version |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-998">BOOKKEEPER-998</a>] - Increased the max entry size to 5MB |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1001">BOOKKEEPER-1001</a>] - Make LocalBookiesRegistry.isLocalBookie() public |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1002">BOOKKEEPER-1002</a>] - BookieRecoveryTest can run out of file descriptors |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1003">BOOKKEEPER-1003</a>] - Fix TestDiskChecker so it can be used on /dev/shm |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1004">BOOKKEEPER-1004</a>] - Allow bookie garbage collection to be triggered manually from tests |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1007">BOOKKEEPER-1007</a>] - Explicit LAC: make the interval configurable in milliseconds instead of seconds |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1008">BOOKKEEPER-1008</a>] - Move to netty4 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1010">BOOKKEEPER-1010</a>] - Bump up Guava version to 20.0 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1022">BOOKKEEPER-1022</a>] - Make BookKeeperAdmin implement AutoCloseable |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1039">BOOKKEEPER-1039</a>] - bk-merge-pr.py ask to run findbugs and rat before merge |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1046">BOOKKEEPER-1046</a>] - Avoid long to Long conversion in OrderedSafeExecutor task submit |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1048">BOOKKEEPER-1048</a>] - Use ByteBuf in LedgerStorageInterface |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1050">BOOKKEEPER-1050</a>] - Cache journalFormatVersionToWrite when starting Journal |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1051">BOOKKEEPER-1051</a>] - Fast shutdown for GarbageCollectorThread |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1052">BOOKKEEPER-1052</a>] - Print autorecovery enabled or not in bookie shell |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1053">BOOKKEEPER-1053</a>] - Upgrade RAT maven version to 0.12 and ignore Eclipse project files |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1055">BOOKKEEPER-1055</a>] - Optimize handling of masterKey in case it is empty |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1056">BOOKKEEPER-1056</a>] - Removed PacketHeader serialization/deserialization allocation |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1063">BOOKKEEPER-1063</a>] - Use executure.execute() instead of submit() to avoid creation of unused FutureTask |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1066">BOOKKEEPER-1066</a>] - Introduce GrowableArrayBlockingQueue |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1068">BOOKKEEPER-1068</a>] - Expose ByteBuf in LedgerEntry to avoid data copy |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1069">BOOKKEEPER-1069</a>] - If client uses V2 proto, set the connection to always decode V2 messages |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1083">BOOKKEEPER-1083</a>] - Improvements on OrderedSafeExecutor |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1084">BOOKKEEPER-1084</a>] - Make variables finale if necessary |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1085">BOOKKEEPER-1085</a>] - Introduce the AlertStatsLogger |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1090">BOOKKEEPER-1090</a>] - Use LOG.isDebugEnabled() to avoid unexpected allocations |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1096">BOOKKEEPER-1096</a>] - When ledger is deleted, along with leaf node all the eligible branch nodes also should be deleted in ZooKeeper. |
| </li> |
| </ul> |
| |
| <h4 id="new-feature">New Feature</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-390">BOOKKEEPER-390</a>] - Provide support for ZooKeeper authentication |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-391">BOOKKEEPER-391</a>] - Support Kerberos authentication of bookkeeper |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-575">BOOKKEEPER-575</a>] - Bookie SSL support |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-670">BOOKKEEPER-670</a>] - Longpoll Read & Piggyback Support |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-912">BOOKKEEPER-912</a>] - Allow EnsemblePlacementPolicy to choose bookies using ledger custom data (multitenancy support) |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-928">BOOKKEEPER-928</a>] - Add custom client supplied metadata field to LedgerMetadata |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-930">BOOKKEEPER-930</a>] - Option to disable Bookie networking |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-941">BOOKKEEPER-941</a>] - Introduce Feature Switches For controlling client and server behavior |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-948">BOOKKEEPER-948</a>] - Provide an option to add more ledger/index directories to a bookie |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-950">BOOKKEEPER-950</a>] - Ledger placement policy to accomodate different storage capacity of bookies |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-969">BOOKKEEPER-969</a>] - Security Support |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-983">BOOKKEEPER-983</a>] - BookieShell Command for LedgerDelete |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-991">BOOKKEEPER-991</a>] - bk shell - Get a list of all on disk files |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-992">BOOKKEEPER-992</a>] - ReadLog Command Enhancement |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1019">BOOKKEEPER-1019</a>] - Support for reading entries after LAC (causal consistency driven by out-of-band communications) |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1034">BOOKKEEPER-1034</a>] - When all disks are full, start Bookie in RO mode if RO mode is enabled |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1067">BOOKKEEPER-1067</a>] - Add Prometheus stats provider |
| </li> |
| </ul> |
| |
| <h4 id="story">Story</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-932">BOOKKEEPER-932</a>] - Move to JDK 8 |
| </li> |
| </ul> |
| |
| <h4 id="task">Task</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-931">BOOKKEEPER-931</a>] - Update the committers list on website |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-996">BOOKKEEPER-996</a>] - Apache Rat Check Failures |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1012">BOOKKEEPER-1012</a>] - Shade and relocate Guava |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1027">BOOKKEEPER-1027</a>] - Cleanup main README and main website page |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1038">BOOKKEEPER-1038</a>] - Fix findbugs warnings and upgrade to 3.0.4 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1043">BOOKKEEPER-1043</a>] - Upgrade Apache Parent Pom Reference to latest version |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1054">BOOKKEEPER-1054</a>] - Add gitignore file |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1059">BOOKKEEPER-1059</a>] - Upgrade to SLF4J-1.7.25 |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1060">BOOKKEEPER-1060</a>] - Add utility to use SafeRunnable from Java8 Lambda |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1070">BOOKKEEPER-1070</a>] - bk-merge-pr.py use apache-rat:check goal instead of rat:rat |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1091">BOOKKEEPER-1091</a>] - Remove Hedwig from BookKeeper website page |
| </li> |
| </ul> |
| |
| <h4 id="test">Test</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-967">BOOKKEEPER-967</a>] - Create new testsuite for testing RackAwareEnsemblePlacementPolicy using ScriptBasedMapping. |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1045">BOOKKEEPER-1045</a>] - Execute tests in different JVM processes |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1064">BOOKKEEPER-1064</a>] - ConcurrentModificationException in AuditorLedgerCheckerTest |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1078">BOOKKEEPER-1078</a>] - Local BookKeeper enhancements for testability |
| </li> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-1097">BOOKKEEPER-1097</a>] - GC test when no WritableDirs |
| </li> |
| </ul> |
| |
| <h4 id="wish">Wish</h4> |
| <ul> |
| <li>[<a href="https://issues.apache.org/jira/browse/BOOKKEEPER-943">BOOKKEEPER-943</a>] - Reduce log level of AbstractZkLedgerManager for register/unregister ReadOnlyLedgerHandle |
| </li> |
| </ul> |
| |
| <h3 id="github">Github</h3> |
| |
| <ul> |
| <li><a href="https://github.com/apache/bookkeeper/milestone/1">https://github.com/apache/bookkeeper/milestone/1</a></li> |
| </ul> |
| |
| </section> |
| |
| |
| </div> |
| </div> |
| |
| <div class="column is-2 is-hidden-mobile"> |
| |
| |
| <div class="toc"> |
| <h2 class="title">Apache BookKeeper 4.5.0 Release Notes</h2> |
| <ul class="section-nav"> |
| <li class="toc-entry toc-h2"><a href="#highlights">Highlights</a> |
| <ul> |
| <li class="toc-entry toc-h3"><a href="#dependencies-upgrade">Dependencies Upgrade</a></li> |
| <li class="toc-entry toc-h3"><a href="#security">Security</a></li> |
| <li class="toc-entry toc-h3"><a href="#public-api">Public API</a> |
| <ul> |
| <li class="toc-entry toc-h4"><a href="#ledgerhandleadv">LedgerHandleAdv</a></li> |
| <li class="toc-entry toc-h4"><a href="#long-poll">Long Poll</a></li> |
| <li class="toc-entry toc-h4"><a href="#explicit-lac">Explicit LAC</a></li> |
| </ul> |
| </li> |
| <li class="toc-entry toc-h3"><a href="#performance">Performance</a> |
| <ul> |
| <li class="toc-entry toc-h4"><a href="#netty-4-upgrade">Netty 4 Upgrade</a></li> |
| <li class="toc-entry toc-h4"><a href="#delay-ensemble-change">Delay Ensemble Change</a></li> |
| <li class="toc-entry toc-h4"><a href="#parallel-ledger-recovery">Parallel Ledger Recovery</a></li> |
| <li class="toc-entry toc-h4"><a href="#multiple-journals">Multiple Journals</a></li> |
| </ul> |
| </li> |
| <li class="toc-entry toc-h3"><a href="#operations">Operations</a> |
| <ul> |
| <li class="toc-entry toc-h4"><a href="#longhierarchicalledgermanager">LongHierarchicalLedgerManager</a></li> |
| <li class="toc-entry toc-h4"><a href="#weight-based-placement-policy">Weight-based placement policy</a></li> |
| <li class="toc-entry toc-h4"><a href="#customized-ledger-metadata">Customized Ledger Metadata</a></li> |
| <li class="toc-entry toc-h4"><a href="#add-prometheus-stats-provider">Add Prometheus stats provider</a></li> |
| <li class="toc-entry toc-h4"><a href="#add-more-tools-in-bookieshell">Add more tools in BookieShell</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toc-entry toc-h2"><a href="#full-list-of-changes">Full list of changes</a> |
| <ul> |
| <li class="toc-entry toc-h3"><a href="#jira">JIRA</a> |
| <ul> |
| <li class="toc-entry toc-h4"><a href="#sub-task">Sub-task</a></li> |
| <li class="toc-entry toc-h4"><a href="#bug">Bug</a></li> |
| <li class="toc-entry toc-h4"><a href="#improvement">Improvement</a></li> |
| <li class="toc-entry toc-h4"><a href="#new-feature">New Feature</a></li> |
| <li class="toc-entry toc-h4"><a href="#story">Story</a></li> |
| <li class="toc-entry toc-h4"><a href="#task">Task</a></li> |
| <li class="toc-entry toc-h4"><a href="#test">Test</a></li> |
| <li class="toc-entry toc-h4"><a href="#wish">Wish</a></li> |
| </ul> |
| </li> |
| <li class="toc-entry toc-h3"><a href="#github">Github</a></li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| |
| |
| |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div id="entry-popover-html" class="popover-template"> |
| <p>An entry is a sequence of bytes (plus some metadata) written to a BookKeeper ledger. Entries are also known as records.</p> |
| |
| </div> |
| |
| <div id="ledger-popover-html" class="popover-template"> |
| <p>A ledger is a sequence of entries written to BookKeeper. Entries are written sequentially to ledgers and at most once, giving ledgers append-only semantics.</p> |
| |
| </div> |
| |
| <div id="bookie-popover-html" class="popover-template"> |
| <p>A bookie is an individual BookKeeper storage server.</p> |
| |
| <p>Bookies store the content of ledgers and act as a distributed ensemble.</p> |
| |
| </div> |
| |
| <div id="rereplication-popover-html" class="popover-template"> |
| <p>A subsystem that runs in the background on bookies to ensure that ledgers are fully replicated even if one bookie from the ensemble is down.</p> |
| |
| </div> |
| |
| <div id="striping-popover-html" class="popover-template"> |
| <p>Striping is the process of distributing BookKeeper ledgers to sub-groups of bookies rather than to all bookies in a BookKeeper ensemble.</p> |
| |
| <p>Striping is essential to ensuring fast performance.</p> |
| |
| </div> |
| |
| <div id="striped-popover-html" class="popover-template"> |
| <p>Striping is the process of distributing BookKeeper ledgers to sub-groups of bookies rather than to all bookies in a BookKeeper ensemble.</p> |
| |
| <p>Striping is essential to ensuring fast performance.</p> |
| |
| </div> |
| |
| <div id="journal-popover-html" class="popover-template"> |
| <p>A journal file stores BookKeeper transaction logs.</p> |
| |
| </div> |
| |
| <div id="fencing-popover-html" class="popover-template"> |
| <p>When a reader forces a ledger to close, preventing any further entries from being written to the ledger.</p> |
| |
| </div> |
| |
| <div id="record-popover-html" class="popover-template"> |
| <p>A record is a sequence of bytes (plus some metadata) written to a BookKeeper ledger. Records are also known as entries.</p> |
| |
| </div> |
| |
| |
| <script type="text/javascript"> |
| |
| tippy('#entry-popover', { |
| html: '#entry-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#ledger-popover', { |
| html: '#ledger-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#bookie-popover', { |
| html: '#bookie-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#rereplication-popover', { |
| html: '#rereplication-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#striping-popover', { |
| html: '#striping-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#striped-popover', { |
| html: '#striped-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#journal-popover', { |
| html: '#journal-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#fencing-popover', { |
| html: '#fencing-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#record-popover', { |
| html: '#record-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| </script> |
| |
| </main> |
| |
| <footer class="footer"> |
| <div class="container"> |
| <div class="content has-text-centered"> |
| <p> |
| Copyright © 2016 - 2021 <a href="https://www.apache.org/">The Apache Software Foundation</a>,<br /> licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, version 2.0</a>. |
| </p> |
| <p> |
| Apache BookKeeper, BookKeeper®, Apache®, the Apache feature logo, and the Apache BookKeeper logo are either registered trademarks or trademarks of The Apache Software Foundation. |
| </p> |
| </div> |
| </div> |
| </footer> |
| |
| </body> |
| |
| <script src="/js/app.js"></script> |
| |
| |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| http://www.apache.org/licenses/LICENSE-2.0 |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License. |
| --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-104419626-1', 'auto'); |
| ga('send', 'pageview'); |
| |
| </script> |
| |
| |
| </html> |
