blob: 4a2ebd74ae61f4e544dd4e16ce29cac9435b449c [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Apache DistributedLog (incubating)</title>
<meta name="description" content="Apache DistributedLog is an high performance replicated log.
">
<link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css">
<link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css">
<!-- JQuery -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script>
<link rel="canonical" href="http://bookkeeper.apache.org/distributedlog/docs/0.4.0-incubating/admin_guide/operations.html" data-proofer-ignore>
<link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://bookkeeper.apache.org/distributedlog/docs/0.4.0-incubating/feed.xml">
<!-- Font Awesome -->
<script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script>
<!-- Google Analytics -->
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-83870961-1', 'auto');
ga('send', 'pageview');
</script>
<!-- End Google Analytics -->
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
</head>
<body role="document">
<nav class="navbar navbar-default navbar-fixed-top">
<div class="container">
<div class="navbar-header">
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png">
</a>
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div id="navbar" class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<!-- Overview -->
<li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li>
<!-- Concepts -->
<li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li>
<!-- Quick Start -->
<li>
<a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li>
<a href="/docs/0.4.0-incubating/start/building.html">
Build DistributedLog from Source
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/start/download.html">
Download Releases
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Quickstart</strong></li>
<li>
<a href="/docs/0.4.0-incubating/start/quickstart.html">
Setup & Run Example
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/basic-1.html">
API - Write Records (via core library)
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/basic-2.html">
API - Write Records (via write proxy)
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/basic-5.html">
API - Read Records
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Deployment</strong></li>
<li>
<a href="/docs/0.4.0-incubating/deployment/cluster.html">
Cluster Setup
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/deployment/global-cluster.html">
Global Cluster Setup
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/deployment/docker.html">
Docker
</a>
</li>
</ul>
</li>
<!-- API -->
<li>
<a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="/docs/0.4.0-incubating/api/java">Java</a></li>
</ul>
</li>
<!-- User Guide -->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
<a href="/docs/0.4.0-incubating/basics/introduction.html">
Introduction
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/considerations/main.html">
Considerations
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/architecture/main.html">
Architecture
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/api/main.html">
API
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/configuration/main.html">
Configuration
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/design/main.html">
Detail Design
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html">
Global Replicated Log
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/implementation/main.html">
Implementation
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/user_guide/references/main.html">
References
</a>
</li>
</ul>
</li>
<!-- Admin Guide -->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/operations.html">
Operations
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/loadtest.html">
Load Test
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/performance.html">
Performance Tuning
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/hardware.html">
Hardware
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/monitoring.html">
Monitoring
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html">
ZooKeeper
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html">
BookKeeper
</a>
</li>
</ul>
</li>
<!-- Tutorials -->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a>
<ul class="dropdown-menu">
<li class="dropdown-header"><strong>Basic</strong></li>
<li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li>
<li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li>
<li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li>
<li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li>
<li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li>
<li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Messaging</strong></li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/messaging-1.html">
Write records to partitioned streams
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/messaging-2.html">
Write records to multiple streams (load balancer)
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/messaging-3.html">
At-least-once Processing
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/messaging-4.html">
Exact-Once Processing
</a>
</li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/messaging-5.html">
Implement a kafka-like pub/sub system
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Replicated State Machines</strong></li>
<li>
<a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html">
Build replicated state machines
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Analytics</strong></li>
<li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li>
</ul>
</li>
</ul>
</div><!--/.nav-collapse -->
</div>
</nav>
<link rel="stylesheet" href="">
<div class="container" role="main">
<div class="row">
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<div class="row">
<!-- Sub Navigation -->
<div class="col-sm-3">
<ul id="sub-nav">
<li><a href="/docs/0.4.0-incubating/admin_guide/main.html" class="">Admin Guide</a>
<ul>
<li>
<a href="/docs/0.4.0-incubating/deployment/cluster.html" class="">
Cluster Setup
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/deployment/global-cluster.html" class="">
Global Cluster Setup
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/operations.html" class="active">
Operations
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/loadtest.html" class="">
Load Test
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/performance.html" class="">
Performance Tuning
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/hardware.html" class="">
Hardware
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/monitoring.html" class="">
Monitoring
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html" class="">
ZooKeeper
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html" class="">
BookKeeper
</a>
<ul>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<!-- Main -->
<div class="col-sm-9">
<!-- Top anchor -->
<a href="#top"></a>
<!-- Breadcrumbs above the main heading -->
<ol class="breadcrumb">
<li><a href="/docs/0.4.0-incubating/admin_guide/main.html">Admin Guide</a></li>
<li class="active">Operations</li>
</ol>
<div class="text">
<!-- Content -->
<div class="contents topic" id="distributedlog-operations">
<p class="topic-title first">DistributedLog Operations</p>
<ul class="simple">
<li><a class="reference internal" href="#id1" id="id2">DistributedLog Operations</a><ul>
<li><a class="reference internal" href="#feature-provider" id="id3">Feature Provider</a><ul>
<li><a class="reference internal" href="#api" id="id4">API</a></li>
<li><a class="reference internal" href="#configuration-based-feature-provider" id="id5">Configuration Based Feature Provider</a></li>
<li><a class="reference internal" href="#available-features" id="id6">Available Features</a></li>
</ul>
</li>
<li><a class="reference internal" href="#dlog" id="id7"><cite>dlog</cite></a><ul>
<li><a class="reference internal" href="#create-a-stream" id="id8">Create a stream</a></li>
<li><a class="reference internal" href="#list-the-streams" id="id9">List the streams</a></li>
<li><a class="reference internal" href="#show-stream-s-information" id="id10">Show stream's information</a></li>
<li><a class="reference internal" href="#dump-a-stream" id="id11">Dump a stream</a></li>
<li><a class="reference internal" href="#delete-a-stream" id="id12">Delete a stream</a></li>
<li><a class="reference internal" href="#truncate-a-stream" id="id13">Truncate a stream</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id1">
<h2><a class="toc-backref" href="#id2">DistributedLog Operations</a></h2>
<div class="section" id="feature-provider">
<h3><a class="toc-backref" href="#id3">Feature Provider</a></h3>
<p>DistributedLog uses a <cite>feature-provider</cite> library provided by Apache BookKeeper for managing features
dynamically at runtime. It is a <a class="reference external" href="https://en.wikipedia.org/wiki/Feature_toggle">feature-flag</a> system used to proportionally control what features
are enabled for the system. In other words, it is a way of altering the control in a system without
restarting it. It can be used during all stages of development, its most visible use case is on
production. For instance, during a production release, you can enable or disable individual features,
control the data flow through the system, thereby minimizing risk of system failure in real time.</p>
<p>This <cite>feature-provider</cite> interface is pluggable and easy to integrate with any configuration management
system.</p>
<div class="section" id="api">
<h4><a class="toc-backref" href="#id4">API</a></h4>
<p><cite>FeatureProvider</cite> is a provider that manages features under different scopes. The provider is responsible
for loading features dynamically at runtime. A <cite>Feature</cite> is a numeric flag that control how much percentage
of this feature will be available to the system - the number is called <cite>availability</cite>.</p>
<pre class="literal-block">
Feature.name() =&gt; returns the name of this feature
Feature.availability() =&gt; returns the availability of this feature
Feature.isAvailable() =&gt; returns true if its availability is larger than 0; otherwise false
</pre>
<p>It is easy to obtain a feature from the provider by just providing a feature name.</p>
<pre class="literal-block">
FeatureProvider provider = ...;
Feature feature = provider.getFeature(&quot;feature1&quot;); // returns the feature named 'feature1'
</pre>
<p>The <cite>FeatureProvider</cite> is scopable to allow creating features in a hierarchical way. For example, if a system
is comprised of two subsystems, one is <em>cache</em>, while the other one is <em>storage</em>. so the features belong to
different subsystems can be created under different scopes.</p>
<pre class="literal-block">
FeatureProvider provider = ...;
FeatureProvider cacheFeatureProvider = provider.scope(&quot;cache&quot;);
FeatureProvider storageFeatureProvider = provider.scope(&quot;storage&quot;);
Feature writeThroughFeature = cacheFeatureProvider.getFeature(&quot;write_through&quot;);
Feature duralWriteFeature = storageFeatureProvider.getFeature(&quot;dural_write&quot;);
// so the available features under `provider` are: (assume scopes are separated by '.')
// - 'cache.write_through'
// - 'storage.dural_write'
</pre>
<p>The feature provider could be passed to <cite>DistributedLogNamespaceBuilder</cite> when building the namespace,
thereby it would be used for controlling the features exposed under <cite>DistributedLogNamespace</cite>.</p>
<pre class="literal-block">
FeatureProvider rootProvider = ...;
FeatureProvider dlFeatureProvider = rootProvider.scope(&quot;dlog&quot;);
DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder()
.uri(uri)
.conf(conf)
.featureProvider(dlFeatureProvider)
.build();
</pre>
<p>The feature provider is loaded by reflection on distributedlog write proxy server. You could specify
the feature provider class name as below. Otherwise it would use <cite>DefaultFeatureProvider</cite>, which disables
all the features by default.</p>
<pre class="literal-block">
featureProviderClass=org.apache.distributedlog.feature.DynamicConfigurationFeatureProvider
</pre>
</div>
<div class="section" id="configuration-based-feature-provider">
<h4><a class="toc-backref" href="#id5">Configuration Based Feature Provider</a></h4>
<p>Beside <cite>DefaultFeatureProvider</cite>, distributedlog also provides a file-based feature provider - it loads
the features from properties files.</p>
<p>All the features and their availabilities are configured in properties file format. For example,</p>
<pre class="literal-block">
cache.write_through=100
storage.dural_write=0
</pre>
<p>You could configure <cite>featureProviderClass</cite> in distributedlog configuration file by setting it to
<cite>org.apache.distributedlog.feature.DynamicConfigurationFeatureProvider</cite> to enable file-based feature
provider. The feature provider will load the features from two files, one is base config file configured
by <cite>fileFeatureProviderBaseConfigPath</cite>, while the other one is overlay config file configured by
<cite>fileFeatureProviderOverlayConfigPath</cite>. Current implementation doesn't differentiate these two files
too much other than the <cite>overlay</cite> config will override the settings in <cite>base</cite> config. It is recommended
to have a base config file for storing the default availability values for your system and dynamically
adjust the availability values in overlay config file.</p>
<pre class="literal-block">
featureProviderClass=org.apache.distributedlog.feature.DynamicConfigurationFeatureProvider
fileFeatureProviderBaseConfigPath=/path/to/base/config
fileFeatureProviderOverlayConfigPath=/path/to/overlay/config
// how frequent we reload the config files
dynamicConfigReloadIntervalSec=60
</pre>
</div>
<div class="section" id="available-features">
<h4><a class="toc-backref" href="#id6">Available Features</a></h4>
<p>Check the <a class="reference external" href="../user_guide/references/features">Features</a> reference page for the features exposed by DistributedLog.</p>
</div>
</div>
<div class="section" id="dlog">
<h3><a class="toc-backref" href="#id7"><cite>dlog</cite></a></h3>
<p>A CLI is provided for inspecting DistributedLog streams and metadata.</p>
<pre class="code bash literal-block">
dlog
JMX enabled by default
Usage: dlog &lt;command&gt;
where <span class="name builtin">command</span> is one of:
<span class="name builtin">local</span> Run distributedlog sandbox
example Run distributedlog example
tool Run distributedlog tool
proxy_tool Run distributedlog proxy tool to interact with proxies
balancer Run distributedlog balancer
admin Run distributedlog admin tool
<span class="name builtin">help</span> This <span class="name builtin">help</span> message
or <span class="name builtin">command</span> is the full name of a class with a defined main<span class="operator">()</span> method.
Environment variables:
DLOG_LOG_CONF Log4j configuration file <span class="operator">(</span>default <span class="name variable">$HOME</span>/src/distributedlog/distributedlog-service/conf/log4j.properties<span class="operator">)</span>
DLOG_EXTRA_OPTS Extra options to be passed to the jvm
DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath
</pre>
<p>These variable can also be set in conf/dlogenv.sh</p>
<div class="section" id="create-a-stream">
<h4><a class="toc-backref" href="#id8">Create a stream</a></h4>
<p>To create a stream:</p>
<pre class="code bash literal-block">
dlog tool create -u &lt;DL URI&gt; -r &lt;STREAM PREFIX&gt; -e &lt;STREAM EXPRESSION&gt;
</pre>
</div>
<div class="section" id="list-the-streams">
<h4><a class="toc-backref" href="#id9">List the streams</a></h4>
<p>To list all the streams under a given DistributedLog namespace:</p>
<pre class="code bash literal-block">
dlog tool list -u &lt;DL URI&gt;
</pre>
</div>
<div class="section" id="show-stream-s-information">
<h4><a class="toc-backref" href="#id10">Show stream's information</a></h4>
<p>To view the metadata associated with a stream:</p>
<pre class="code bash literal-block">
dlog tool show -u &lt;DL URI&gt; -s &lt;STREAM NAME&gt;
</pre>
</div>
<div class="section" id="dump-a-stream">
<h4><a class="toc-backref" href="#id11">Dump a stream</a></h4>
<p>To dump the items inside a stream:</p>
<pre class="code bash literal-block">
dlog tool dump -u &lt;DL URI&gt; -s &lt;STREAM NAME&gt; -o &lt;START TXN ID&gt; -l &lt;NUM RECORDS&gt;
</pre>
</div>
<div class="section" id="delete-a-stream">
<h4><a class="toc-backref" href="#id12">Delete a stream</a></h4>
<p>To delete a stream, run:</p>
<pre class="code bash literal-block">
dlog tool delete -u &lt;DL URI&gt; -s &lt;STREAM NAME&gt;
</pre>
</div>
<div class="section" id="truncate-a-stream">
<h4><a class="toc-backref" href="#id13">Truncate a stream</a></h4>
<p>Truncate the streams under a given DistributedLog namespace. You could specify a filter to match the streams that you want to truncate.</p>
<p>There is a difference between the <tt class="docutils literal">truncate</tt> and <tt class="docutils literal">delete</tt> command. When you issue a <tt class="docutils literal">truncate</tt>, the data will be purge without removing the streams. A <tt class="docutils literal">delete</tt> will delete the stream. You can pass the flag <tt class="docutils literal"><span class="pre">-delete</span></tt> to the <tt class="docutils literal">truncate</tt> command to also delete the streams.</p>
<pre class="code bash literal-block">
dlog tool truncate -u &lt;DL URI&gt;
</pre>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<hr>
<div class="row">
<div class="col-xs-12">
<footer>
<p class="text-center">&copy; Copyright 2016
<a href="http://www.apache.org">The Apache Software Foundation.</a> All Rights Reserved.
</p>
<p class="text-center">
<a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a>
</p>
</footer>
</div>
</div>
<!-- container div end -->
</div>
<script>
(function () {
'use strict';
anchors.options.placement = 'right';
anchors.add();
})();
</script>
</body>
</html>