| <!DOCTYPE html> |
| <html> |
| <head> |
| <title>Apache BookKeeper™ - DistributedLog</title> |
| |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| |
| <link rel="stylesheet" href="/css/normalize.css"> |
| <link rel="stylesheet" href="/css/tippy.css"> |
| <link rel="stylesheet" href="/css/style.css"> |
| |
| <link rel="shortcut icon" href="/img/favicon.ico"> |
| |
| <script src="/js/tippy.min.js"></script> |
| |
| <script type="text/javascript"> |
| var shiftWindow = function() { scrollBy(0, -25); }; |
| window.addEventListener("hashchange", shiftWindow); |
| window.addEventListener("pageshow", shiftWindow); |
| function load() { if (window.location.hash) shiftWindow(); } |
| </script> |
| </head> |
| <body class="body"> |
| <main class="main"> |
| |
| <nav class="navbar bk-topnav"> |
| <div class="navbar-brand"> |
| <a class="navbar-item bk-brand" href="/"> |
| Apache BookKeeper™ |
| </a> |
| |
| <div class="navbar-burger burger" data-target="bkNav"> |
| <span></span> |
| <span></span> |
| <span></span> |
| </div> |
| </div> |
| |
| <div id="bkNav" class="navbar-menu"> |
| <div class="navbar-start"> |
| <div class="navbar-item has-dropdown is-hoverable"> |
| <a class="navbar-link">Documentation</a> |
| <div class="navbar-dropdown is-boxed"> |
| <a class="navbar-item" href="/docs/latest/overview/overview"> |
| Version 4.15.0-SNAPSHOT |
| <span class="tag is-warning">Development</span> |
| </a> |
| <a class="navbar-item" href="/docs/latest/api/javadoc"> |
| <span class="icon bk-javadoc-icon"> |
| <img src="/img/java-icon.svg"> |
| </span> |
| Javadoc |
| </a> |
| <hr class="dropdown-divider"> |
| |
| <a class="navbar-item" href="/docs/4.14.0/overview/overview"> |
| Release 4.14.0 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.13.0/overview/overview"> |
| Release 4.13.0 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.12.1/overview/overview"> |
| Release 4.12.1 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.12.0/overview/overview"> |
| Release 4.12.0 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.11.1/overview/overview"> |
| Release 4.11.1 |
| |
| <span class="tag is-success">Stable</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.11.0/overview/overview"> |
| Release 4.11.0 |
| |
| </a> |
| |
| <a class="navbar-item" href="/docs/4.10.0/overview/overview"> |
| Release 4.10.0 |
| |
| </a> |
| |
| |
| <a class="navbar-item" href="/archives/docs/r4.9.2"> |
| Release 4.9.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.9.1"> |
| Release 4.9.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.9.0"> |
| Release 4.9.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.8.2"> |
| Release 4.8.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.8.1"> |
| Release 4.8.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.8.0"> |
| Release 4.8.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.3"> |
| Release 4.7.3 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.2"> |
| Release 4.7.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.1"> |
| Release 4.7.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.7.0"> |
| Release 4.7.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.6.2"> |
| Release 4.6.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.6.1"> |
| Release 4.6.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.6.0"> |
| Release 4.6.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.5.1"> |
| Release 4.5.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.5.0"> |
| Release 4.5.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.4.0"> |
| Release 4.4.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.3.2"> |
| Release 4.3.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.3.1"> |
| Release 4.3.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.3.0"> |
| Release 4.3.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.4"> |
| Release 4.2.4 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.3"> |
| Release 4.2.3 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.2"> |
| Release 4.2.2 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.1"> |
| Release 4.2.1 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.2.0"> |
| Release 4.2.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.1.0"> |
| Release 4.1.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| <a class="navbar-item" href="/archives/docs/r4.0.0"> |
| Release 4.0.0 |
| |
| <span class="tag is-warning">EOL</span> |
| |
| </a> |
| |
| </div> |
| </div> |
| |
| <div class="navbar-item has-dropdown is-hoverable"> |
| <a class="navbar-link">Community</a> |
| <div class="navbar-dropdown is-boxed"> |
| <a class="navbar-item" href="/community/mailing-lists">Mailing lists</a> |
| <a class="navbar-item" href="/community/slack">Slack</a> |
| <a class="navbar-item" href="https://github.com/apache/bookkeeper/issues">Github Issues</a> |
| <a class="navbar-item" href="/community/releases">Release Management</a> |
| <a class="navbar-item" href="/community/meeting">Community Meetings</a> |
| <hr class="dropdown-divider"> |
| <a class="navbar-item" href="/community/contributing">Contribution Guide</a> |
| <a class="navbar-item" href="/community/coding_guide">Coding Guide</a> |
| <a class="navbar-item" href="/community/testing">Testing Guide</a> |
| <a class="navbar-item" href="/community/issue-report">Issue Report Guide</a> |
| <a class="navbar-item" href="/community/release_guide">Release Guide</a> |
| <hr class="dropdown-divider"> |
| <a class="navbar-item" href="/community/presentations">Presentations</a> |
| <a class="navbar-item" href="/community/bookkeeper_proposals">BookKeeper Proposals</a> |
| </div> |
| </div> |
| |
| <div class="navbar-item has-dropdown is-hoverable"> |
| <a class="navbar-link">Project</a> |
| <div class="navbar-dropdown is-boxed"> |
| <a class="navbar-item" href="/project/who">Who are we?</a> |
| <a class="navbar-item" href="/project/bylaws">Bylaws</a> |
| <a class="navbar-item" href="http://www.apache.org/licenses/">License</a> |
| <hr class="dropdown-divider"> |
| <a class="navbar-item" href="/project/privacy">Privacy policy</a> |
| <a class="navbar-item" href="http://www.apache.org/foundation/sponsorship.html">Sponsorship</a> |
| <a class="navbar-item" href="http://www.apache.org/foundation/thanks.html">Thanks</a> |
| </div> |
| </div> |
| </div> |
| |
| <div class="navbar-end"> |
| <div class="navbar-item"> |
| <div class="field is-grouped"> |
| <p class="control"> |
| <a class="button bk-twitter" href="https://twitter.com/asfbookkeeper"> |
| <span class="icon"> |
| <i class="fa fa-twitter"></i> |
| </span> |
| <span>Twitter</span> |
| </a> |
| </p> |
| <p class="control"> |
| <a class="button" href="https://github.com/apache/bookkeeper"> |
| <span class="icon"> |
| <i class="fa fa-github"></i> |
| </span> |
| <span>GitHub</span> |
| </a> |
| </p> |
| <p class="control"> |
| <a class="button is-primary" href="/releases"> |
| <span class="icon"> |
| <i class="fa fa-download"></i> |
| </span> |
| <span>Download</span> |
| </a> |
| </p> |
| </div> |
| </div> |
| </div> |
| </div> |
| </nav> |
| |
| |
| <div class="bk-docs-container"> |
| <div class="columns is-gapless"> |
| <div class="column is-2 is-hidden-mobile"> |
| <div class="container"> |
| |
| <aside class="sidebar"> |
| |
| <a class="button is-info"> |
| Version: 4.10.0 |
| </a> |
| <hr /> |
| |
| <p> |
| Getting started |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../getting-started/installation"> |
| Installation |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../getting-started/run-locally"> |
| Run bookies locally |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../getting-started/concepts"> |
| Concepts and architecture |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Deployment |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../deployment/manual"> |
| Manual deployment |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../deployment/dcos"> |
| BookKeeper on DC/OS |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../deployment/kubernetes"> |
| BookKeeper on Kubernetes |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Administration |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../admin/bookies"> |
| BookKeeper administration |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/autorecovery"> |
| AutoRecovery |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/metrics"> |
| Metric collection |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/upgrade"> |
| Upgrade |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/http"> |
| BookKeeper Admin REST API |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../admin/decomission"> |
| Decommissioning Bookies |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| API |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../api/overview"> |
| Overview |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/ledger-api"> |
| Ledger API |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/ledger-adv-api"> |
| Advanced Ledger API |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/distributedlog-api"> |
| DistributedLog |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../api/javadoc"> |
| Java API Docs |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Security |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../security/overview"> |
| Overview |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../security/tls"> |
| TLS Authentication |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../security/sasl"> |
| SASL Authentication |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../security/zookeeper"> |
| ZooKeeper Authentication |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Development |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../development/protocol"> |
| BookKeeper protocol |
| </a> |
| </li> |
| |
| </ul> |
| |
| <p> |
| Reference |
| </p> |
| <ul class="sidebar-items"> |
| |
| |
| <li> |
| <a href="../../reference/config"> |
| Configuration |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../reference/cli"> |
| Command-line tools |
| </a> |
| </li> |
| |
| |
| <li> |
| <a href="../../reference/metrics"> |
| Metrics |
| </a> |
| </li> |
| |
| </ul> |
| |
| </aside> |
| |
| |
| </div> |
| </div> |
| |
| <div class="column is-8 bk-docs-block"> |
| <header class="docs-title"> |
| <nav class="level bk-level"> |
| <div class="level-left"> |
| <div class="level-item"> |
| <h1 class="title">DistributedLog</h1> |
| </div> |
| </div> |
| |
| </nav> |
| |
| <h2 class="subtitle">A higher-level API for managing BookKeeper entries</h2> |
| </header> |
| |
| <hr /> |
| |
| <div class="content"> |
| <section class="bk-main-content"> |
| <blockquote> |
| <p>DistributedLog began its life as a separate project under the Apache Foundation. It was merged into BookKeeper in 2017.</p> |
| </blockquote> |
| |
| <p>The DistributedLog API is an easy-to-use interface for managing BookKeeper entries that enables you to use BookKeeper without needing to interact with <a href="../ledger-api">ledgers</a> directly.</p> |
| |
| <p>DistributedLog (DL) maintains sequences of records in categories called <em>logs</em> (aka <em>log streams</em>). <em>Writers</em> append records to DL logs, while <em>readers</em> fetch and process those records.</p> |
| |
| <h2 id="architecture">Architecture</h2> |
| |
| <p>The diagram below illustrates how the DistributedLog API works with BookKeeper:</p> |
| |
| <p><img src="/img/distributedlog.png" alt="DistributedLog API" /></p> |
| |
| <h2 id="logs">Logs</h2> |
| |
| <p>A <em>log</em> in DistributedLog is an ordered, immutable sequence of <em>log records</em>.</p> |
| |
| <p>The diagram below illustrates the anatomy of a log stream:</p> |
| |
| <p><img src="/img/logs.png" alt="DistributedLog log" /></p> |
| |
| <h3 id="log-records">Log records</h3> |
| |
| <p>Each log record is a sequence of bytes. Applications are responsible for serializing and deserializing byte sequences stored in log records.</p> |
| |
| <p>Log records are written sequentially into a <em>log stream</em> and assigned with a a unique sequence number called a DLSN (<strong>D</strong>istributed<strong>L</strong>og <strong>S</strong>equence <strong>N</strong>umber).</p> |
| |
| <p>In addition to a DLSN, applications can assign their own sequence number when constructing log records. Application-defined sequence numbers are known as <em>TransactionIDs</em> (or <em>txid</em>). Either a DLSN or a TransactionID can be used for positioning readers to start reading from a specific log record.</p> |
| |
| <h3 id="log-segments">Log segments</h3> |
| |
| <p>Each log is broken down into <em>log segments</em> that contain subsets of records. Log segments are distributed and stored in BookKeeper. DistributedLog rolls the log segments based on the configured <em>rolling policy</em>, which be either</p> |
| |
| <ul> |
| <li>a configurable period of time (such as every 2 hours), or</li> |
| <li>a configurable maximum size (such as every 128 MB).</li> |
| </ul> |
| |
| <p>The data in logs is divided up into equally sized log segments and distributed evenly across <span class="pop" id="bookie-popover">bookies</span>. This allows logs to scale beyond a size that would fit on a single server and spreads read traffic across the cluster.</p> |
| |
| <h3 id="namespaces">Namespaces</h3> |
| |
| <p>Log streams that belong to the same organization are typically categorized and managed under a <em>namespace</em>. DistributedLog namespaces essentially enable applications to locate log streams. Applications can perform the following actions under a namespace:</p> |
| |
| <ul> |
| <li>create streams</li> |
| <li>delete streams</li> |
| <li>truncate streams to a given sequence number (either a DLSN or a TransactionID)</li> |
| </ul> |
| |
| <h2 id="writers">Writers</h2> |
| |
| <p>Through the DistributedLog API, writers write data into logs of their choice. All records are appended into logs in order. The sequencing is performed by the writer, which means that there is only one active writer for a log at any given time.</p> |
| |
| <p>DistributedLog guarantees correctness when two writers attempt to write to the same log when a network partition occurs using a <em>fencing</em> mechanism in the log segment store.</p> |
| |
| <h3 id="write-proxy">Write Proxy</h3> |
| |
| <p>Log writers are served and managed in a service tier called the <em>Write Proxy</em> (see the diagram <a href="#architecture">above</a>). The Write Proxy is used for accepting writes from a large number of clients.</p> |
| |
| <h2 id="readers">Readers</h2> |
| |
| <p>DistributedLog readers read records from logs of their choice, starting with a provided position. The provided position can be either a DLSN or a TransactionID.</p> |
| |
| <p>Readers read records from logs in strict order. Different readers can read records from different positions in the same log.</p> |
| |
| <p>Unlike other pub-sub systems, DistributedLog doesn’t record or manage readers’ positions. This means that tracking is the responsibility of applications, as different applications may have different requirements for tracking and coordinating positions. This is hard to get right with a single approach. Distributed databases, for example, might store reader positions along with SSTables, so they would resume applying transactions from the positions store in SSTables. Tracking reader positions could easily be done at the application level using various stores (such as ZooKeeper, the filesystem, or key-value stores).</p> |
| |
| <h3 id="read-proxy">Read Proxy</h3> |
| |
| <p>Log records can be cached in a service tier called the <em>Read Proxy</em> to serve a large number of readers. See the diagram <a href="#architecture">above</a>. The Read Proxy is the analogue of the <a href="#write-proxy">Write Proxy</a>.</p> |
| |
| <h2 id="guarantees">Guarantees</h2> |
| |
| <p>The DistributedLog API for BookKeeper provides a number of guarantees for applications:</p> |
| |
| <ul> |
| <li>Records written by a <a href="#writers">writer</a> to a <a href="#logs">log</a> are appended in the order in which they are written. If a record <strong>R1</strong> is written by the same writer as a record <strong>R2</strong>, <strong>R1</strong> will have a smaller sequence number than <strong>R2</strong>.</li> |
| <li><a href="#readers">Readers</a> see <a href="#log-records">records</a> in the same order in which they are <a href="#writers">written</a> to the log.</li> |
| <li>All records are persisted on disk by BookKeeper before acknowledgements, which guarantees durability.</li> |
| <li>For a log with a replication factor of N, DistributedLog tolerates up to N-1 server failures without losing any records.</li> |
| </ul> |
| |
| <h2 id="api">API</h2> |
| |
| <p>Documentation for the DistributedLog API can be found <a href="https://bookkeeper.apache.org/distributedlog/docs/latest/user_guide/api/core">here</a>.</p> |
| |
| <blockquote> |
| <p>At a later date, the DistributedLog API docs will be added here.</p> |
| </blockquote> |
| |
| <!-- |
| |
| The DistributedLog core library is written in Java and interacts with namespaces and logs directly. |
| |
| ### Installation |
| |
| The BookKeeper Java client library is available via [Maven Central](http://search.maven.org/) and can be installed using [Maven](#maven), [Gradle](#gradle), and other build tools. |
| |
| ### Maven |
| |
| If you're using [Maven](https://maven.apache.org/), add this to your [`pom.xml`](https://maven.apache.org/guides/introduction/introduction-to-the-pom.html) build configuration file: |
| |
| ```xml |
| <-- in your <properties> block -> |
| <bookkeeper.version>4.12.1</bookkeeper.version> |
| |
| <-- in your <dependencies> block -> |
| <dependency> |
| <groupId>org.apache.bookkeeper</groupId> |
| <artifactId>bookkeeper-server</artifactId> |
| <version>${bookkeeper.version}</version> |
| </dependency> |
| ``` |
| |
| ### Gradle |
| |
| If you're using [Gradle](https://gradle.org/), add this to your [`build.gradle`](https://spring.io/guides/gs/gradle/) build configuration file: |
| |
| ```groovy |
| dependencies { |
| compile group: 'org.apache.bookkeeper', name: 'bookkeeper-server', version: '4.10.0' |
| } |
| |
| // Alternatively: |
| dependencies { |
| compile 'org.apache.bookkeeper:bookkeeper-server:4.10.0' |
| } |
| ``` |
| |
| ### Namespace API |
| |
| A DL [namespace](#namespace) is a collection of [log streams](#log-streams). When using the DistributedLog API with BookKeeper, you need to provide your Java client with a namespace URI. That URI consists of three elements: |
| |
| 1. The `distributedlog-bk` scheme |
| 1. A connection string for your BookKeeper cluster. You have three options for the connection string: |
| * An entire ZooKeeper connection string, for example `zk1:2181,zk2:2181,zk3:2181` |
| * A host and port for one node in your ZooKeeper cluster, for example `zk1:2181`. In general, it's better to provide a full ZooKeeper connection string. |
| * If your ZooKeeper cluster can be discovered via DNS, you can provide the DNS name, for example `my-zookeeper-cluster.com`. |
| 1. A path that points to the location where logs are stored. This could be a ZooKeeper [znode](https://zookeeper.apache.org/doc/current/zookeeperOver.html). |
| |
| This is the general structure of a namespace URI: |
| |
| ```shell |
| distributedlog-bk://{connection-string}/{path} |
| ``` |
| |
| Here are some example URIs: |
| |
| ```shell |
| distributedlog-bk://zk1:2181,zk2:2181,zk3:2181/my-namespace # Full ZooKeeper connection string |
| distributedlog-bk://localhost:2181/my-namespace # Single ZooKeeper node |
| distributedlog-bk://my-zookeeper-cluster.com/my-namespace # DNS name for ZooKeeper |
| ``` |
| |
| #### Creating namespaces |
| |
| In order to create namespaces, you need to use the command-line tool. |
| |
| ```shell |
| $ |
| ``` |
| |
| #### Using namespaces |
| |
| Once you have a namespace URI, you can build a namespace instance, which will be used for operating streams. Use the `DistributedLogNamespaceBuilder` to build a `DistributedLogNamespace` object, passing in a `DistributedLogConfiguration`, a URI, and optionally a stats logger and a feature provider. |
| |
| ```java |
| DistributedLogConfiguration conf = new DistributedLogConfiguration(); |
| URI uri = URI.create("distributedlog-bk://localhost:2181/my-namespace "); |
| DistributedLogNamespaceBuilder builder = DistributedLogNamespaceBuilder.newBuilder(); |
| DistributedLogNamespace = builder |
| .conf(conf) // Configuration for the namespace |
| .uri(uri) // URI for the namespace |
| .statsLogger(...) // Stats logger for statistics |
| .featureProvider(...) // Feature provider for controlling features |
| .build(); |
| ``` |
| |
| ### Log API |
| |
| #### Creating logs |
| |
| You can create a log by calling the `createLog` method on a `DistributedLogNamespace` object, passing in a name for the log. This creates the log under the namespace but does *not* return a handle for operating the log. |
| |
| ```java |
| DistributedLogNamespace namespace = /* Create namespace */; |
| try { |
| namespace.createLog("test-log"); |
| } catch (IOException e) } |
| // Handle the log creation exception |
| } |
| ``` |
| |
| #### Opening logs |
| |
| A `DistributedLogManager` handle will be returned when opening a log using the `openLog` function, which takes the name of the log. This handle can be used for writing records to or reading records from the log. |
| |
| > If the log doesn't exist and `createStreamIfNotExists` is set to `true` in the configuration, the log will be created automatically when writing the first record. |
| |
| ```java |
| DistributedLogConfiguration conf = new DistributedLogConfiguration(); |
| conf.setCreateStreamIfNotExists(true); |
| DistributedLogNamespace namespace = DistributedLogNamespace.newBuilder() |
| .conf(conf) |
| // Other builder attributes |
| .build(); |
| DistributedLogManager logManager = namespace.openLog("test-log"); |
| ``` |
| |
| Sometimes, applications may open a log with a different configuration from the enclosing namespace. This can be done using the same `openLog` method: |
| |
| ```java |
| // Namespace configuration |
| DistributedLogConfiguration namespaceConf = new DistributedLogConfiguration(); |
| conf.setRetentionPeriodHours(24); |
| URI uri = URI.create("distributedlog-bk://localhost:2181/my-namespace"); |
| DistributedLogNamespace namespace = DistributedLogNamespace.newBuilder() |
| .conf(namespaceConf) |
| .uri(uri) |
| // Other builder attributes |
| .build(); |
| // Log-specific configuration |
| DistributedLogConfiguration logConf = new DistributedLogConfiguration(); |
| logConf.setRetentionPeriodHours(12); |
| DistributedLogManager logManager = namespace.openLog( |
| "test-log", |
| Optional.of(logConf), |
| Optional.absent() |
| ); |
| ``` |
| |
| #### Deleting logs |
| |
| The `DistributedLogNamespace` class provides `deleteLog` function that can be used to delete logs. When you delete a lot, the client library will attempt to acquire a lock on the log before deletion. If the log is being written to by an active writer, deletion will fail (as the other writer currently holds the lock). |
| |
| ```java |
| try { |
| namespace.deleteLog("test-log"); |
| } catch (IOException e) { |
| // Handle exception |
| } |
| ``` |
| |
| #### Checking for the existence of a log |
| |
| Applications can check whether a log exists by calling the `logExists` function. |
| |
| ```java |
| if (namespace.logExists("test-log")) { |
| // Perform some action when the log exists |
| } else { |
| // Perform some action when the log doesn't exist |
| } |
| ``` |
| |
| #### Listing logs |
| |
| Applications can retrieve a list of all logs under a namespace using the `getLogs` function. |
| |
| ```java |
| Iterator<String> logs = namespace.getLogs(); |
| while (logs.hasNext()) { |
| String logName = logs.next(); |
| // Do something with the log name, such as print |
| } |
| ``` |
| |
| ### Writer API |
| |
| You can write to DistributedLog logs either [synchronously](#writing-to-logs-synchronously) using the `LogWriter` class or [asynchronously](#writing-to-logs-asynchronously) using the `AsyncLogWriter` class. |
| |
| #### Immediate flush |
| |
| By default, records are buffered rather than being written immediately. You can disable this behavior and make DL writers write ("flush") entries immediately by adding the following to your configuration object: |
| |
| ```java |
| conf.setImmediateFlushEnabled(true); |
| conf.setOutputBufferSize(0); |
| conf.setPeriodicFlushFrequencyMilliSeconds(0); |
| ``` |
| |
| #### Immediate locking |
| |
| By default, DL writers can write to a log stream when other writers are also writing to that stream. You can override this behavior and disable other writers from writing to the stream by adding this to your configuration: |
| |
| ```java |
| conf.setLockTimeout(DistributedLogConstants.LOCK_IMMEDIATE); |
| ``` |
| |
| #### Writing to logs synchronously |
| |
| To write records to a log synchronously, you need to instantiate a `LogWriter` object using a `DistributedLogManager`. Here's an example: |
| |
| ```java |
| DistributedLogNamespace namespace = /* Some namespace object */; |
| DistributedLogManager logManager = namespace.openLog("test-log"); |
| LogWriter writer = logManager.startLogSegmentNonPartitioned(); |
| ``` |
| |
| > The DistributedLog library enforces single-writer semantics by deploying a ZooKeeper locking mechanism. If there is only one active writer, subsequent calls to `startLogSegmentNonPartitioned` will fail with an `OwnershipAcquireFailedException`. |
| |
| Log records represent the data written to a log stream. Each log record is associated with an application-defined [TransactionID](#log-records). This ID must be non decreasing or else writing a record will be rejected with `TransactionIdOutOfOrderException`. The application is allowed to bypass the TransactionID sanity checking by setting `maxIdSanityCheck` to `false` in the configuration. System time and atomic numbers are good candidates for TransactionID. |
| |
| ```java |
| long txid = 1L; |
| byte[] data = "some byte array".getBytes(); |
| LogRecord record = new LogRecord(txid, data); |
| ``` |
| |
| Your application can write either a single record, using the `write` method, or many records, using the `writeBulk` method. |
| |
| ```java |
| // Single record |
| writer.write(record); |
| |
| // Bulk write |
| List<LogRecord> records = Lists.newArrayList(); |
| records.add(record); |
| writer.writeBulk(records); |
| ``` |
| |
| The write calls return immediately after the records are added into the output buffer of writer. This means that the data isn't guaranteed to be durable until the writer explicitly calls `setReadyToFlush` and `flushAndSync`. Those two calls will first transmit buffered data to the backend, wait for transmit acknowledgements (acks), and commit the written data to make them visible to readers. |
| |
| ```java |
| // Flush the records |
| writer.setReadyToFlush(); |
| |
| // Commit the records to make them visible to readers |
| writer.flushAndSync(); |
| ``` |
| |
| Log streams in DistributedLog are endless streams *unless they are sealed*. Endless in this case means that writers can keep writing records to those streams, readers can keep reading from the end of those streams, and the process never stops. Your application can seal a log stream using the `markEndOfStream` method: |
| |
| ```java |
| writer.markEndOfStream(); |
| ``` |
| |
| #### Writing to logs asynchronously |
| |
| In order to write to DistributedLog logs asynchronously, you need to create an `AsyncLogWriter` instread of a `LogWriter`. |
| |
| ```java |
| DistributedLogNamespace namespace = /* Some namespace object */; |
| DistributedLogManager logManager = namespace.openLog("test-async-log"); |
| AsyncLogWriter asyncWriter = logManager.startAsyncLogSegmentNonPartitioned(); |
| ``` |
| |
| All writes to `AsyncLogWriter` are non partitioned. The futures representing write results are only satisfied when the data is durably persisted in the stream. A [DLSN](#log-records) will be returned for each write, which is used to represent the position (aka offset) of the record in the log stream. All the records added in order are guaranteed to be persisted in order. Here's an example of an async writer that gathers a list of futures representing multiple async write results: |
| |
| ```java |
| List<Future<DLSN>> addFutures = Lists.newArrayList(); |
| for (long txid = 1L; txid <= 100L; txid++) { |
| byte[] data = /* some byte array */; |
| LogRecord record = new LogRecord(txid, data); |
| addFutures.add(asyncWriter.write(record)); |
| } |
| List<DLSN> addResults = Await.result(Future.collect(addFutures)); |
| ``` |
| |
| The `AsyncLogWriter` also provides a method for truncating a stream to a given DLSN. This is useful for building replicated state machines that need explicit controls on when the data can be deleted. |
| |
| ```java |
| DLSN truncateDLSN = /* some DLSN */; |
| Future<DLSN> truncateFuture = asyncWriter.truncate(truncateDLSN); |
| |
| // Wait for truncation result |
| Await.result(truncateFuture); |
| ``` |
| |
| ##### Register a listener |
| |
| Instead of returning a future from write operations, you can also set up a listener that performs assigned actions upon success or failure of the write. Here's an example: |
| |
| ```java |
| asyncWriter.addEventListener(new FutureEventListener<DLSN>() { |
| @Override |
| public void onFailure(Throwable cause) { |
| // Execute if the attempt fails |
| } |
| |
| @Override |
| public void onSuccess(DLSN value) { |
| // Execute if the attempt succeeds |
| } |
| }); |
| ``` |
| |
| ##### Close the writer |
| |
| You can close an async writer when you're finished with it like this: |
| |
| ```java |
| FutureUtils.result(asyncWriter.asyncClose()); |
| ``` |
| |
| <!-- |
| TODO: Reader API |
| --> |
| |
| </section> |
| |
| |
| </div> |
| </div> |
| |
| <div class="column is-2 is-hidden-mobile"> |
| |
| |
| <div class="toc"> |
| <h2 class="title">DistributedLog</h2> |
| <ul class="section-nav"> |
| <li class="toc-entry toc-h2"><a href="#architecture">Architecture</a></li> |
| <li class="toc-entry toc-h2"><a href="#logs">Logs</a> |
| <ul> |
| <li class="toc-entry toc-h3"><a href="#log-records">Log records</a></li> |
| <li class="toc-entry toc-h3"><a href="#log-segments">Log segments</a></li> |
| <li class="toc-entry toc-h3"><a href="#namespaces">Namespaces</a></li> |
| </ul> |
| </li> |
| <li class="toc-entry toc-h2"><a href="#writers">Writers</a> |
| <ul> |
| <li class="toc-entry toc-h3"><a href="#write-proxy">Write Proxy</a></li> |
| </ul> |
| </li> |
| <li class="toc-entry toc-h2"><a href="#readers">Readers</a> |
| <ul> |
| <li class="toc-entry toc-h3"><a href="#read-proxy">Read Proxy</a></li> |
| </ul> |
| </li> |
| <li class="toc-entry toc-h2"><a href="#guarantees">Guarantees</a></li> |
| <li class="toc-entry toc-h2"><a href="#api">API</a></li> |
| </ul> |
| </div> |
| |
| |
| |
| </div> |
| </div> |
| </div> |
| |
| |
| |
| <div id="entry-popover-html" class="popover-template"> |
| <p>An entry is a sequence of bytes (plus some metadata) written to a BookKeeper ledger. Entries are also known as records.</p> |
| |
| </div> |
| |
| <div id="ledger-popover-html" class="popover-template"> |
| <p>A ledger is a sequence of entries written to BookKeeper. Entries are written sequentially to ledgers and at most once, giving ledgers append-only semantics.</p> |
| |
| </div> |
| |
| <div id="bookie-popover-html" class="popover-template"> |
| <p>A bookie is an individual BookKeeper storage server.</p> |
| |
| <p>Bookies store the content of ledgers and act as a distributed ensemble.</p> |
| |
| </div> |
| |
| <div id="rereplication-popover-html" class="popover-template"> |
| <p>A subsystem that runs in the background on bookies to ensure that ledgers are fully replicated even if one bookie from the ensemble is down.</p> |
| |
| </div> |
| |
| <div id="striping-popover-html" class="popover-template"> |
| <p>Striping is the process of distributing BookKeeper ledgers to sub-groups of bookies rather than to all bookies in a BookKeeper ensemble.</p> |
| |
| <p>Striping is essential to ensuring fast performance.</p> |
| |
| </div> |
| |
| <div id="striped-popover-html" class="popover-template"> |
| <p>Striping is the process of distributing BookKeeper ledgers to sub-groups of bookies rather than to all bookies in a BookKeeper ensemble.</p> |
| |
| <p>Striping is essential to ensuring fast performance.</p> |
| |
| </div> |
| |
| <div id="journal-popover-html" class="popover-template"> |
| <p>A journal file stores BookKeeper transaction logs.</p> |
| |
| </div> |
| |
| <div id="fencing-popover-html" class="popover-template"> |
| <p>When a reader forces a ledger to close, preventing any further entries from being written to the ledger.</p> |
| |
| </div> |
| |
| <div id="record-popover-html" class="popover-template"> |
| <p>A record is a sequence of bytes (plus some metadata) written to a BookKeeper ledger. Records are also known as entries.</p> |
| |
| </div> |
| |
| |
| <script type="text/javascript"> |
| |
| tippy('#entry-popover', { |
| html: '#entry-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#ledger-popover', { |
| html: '#ledger-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#bookie-popover', { |
| html: '#bookie-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#rereplication-popover', { |
| html: '#rereplication-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#striping-popover', { |
| html: '#striping-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#striped-popover', { |
| html: '#striped-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#journal-popover', { |
| html: '#journal-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#fencing-popover', { |
| html: '#fencing-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| tippy('#record-popover', { |
| html: '#record-popover-html', |
| arrow: true, |
| animation: 'fade' |
| }); |
| |
| </script> |
| |
| </main> |
| |
| <footer class="footer"> |
| <div class="container"> |
| <div class="content has-text-centered"> |
| <p> |
| Copyright © 2016 - 2021 <a href="https://www.apache.org/">The Apache Software Foundation</a>,<br /> licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, version 2.0</a>. |
| </p> |
| <p> |
| Apache BookKeeper, BookKeeper®, Apache®, the Apache feature logo, and the Apache BookKeeper logo are either registered trademarks or trademarks of The Apache Software Foundation. |
| </p> |
| </div> |
| </div> |
| </footer> |
| |
| </body> |
| |
| <script src="/js/app.js"></script> |
| |
| |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| http://www.apache.org/licenses/LICENSE-2.0 |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License. |
| --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-104419626-1', 'auto'); |
| ga('send', 'pageview'); |
| |
| </script> |
| |
| |
| </html> |
