blob: 33d715570e3d36984d92c6313df9cee90d654aff [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Apache DistributedLog</title>
<meta name="description" content="Apache DistributedLog is an high performance replicated log.
">
<link rel="stylesheet" href="/docs/latest/styles/site.css">
<link rel="stylesheet" href="/docs/latest/css/theme.css">
<!-- JQuery -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
<script src="/docs/latest/js/bootstrap.min.js"></script>
<link rel="canonical" href="http://bookkeeper.apache.org/distributedlog/docs/latest/user_guide/configuration/perlog.html" data-proofer-ignore>
<link rel="alternate" type="application/rss+xml" title="Apache DistributedLog" href="http://bookkeeper.apache.org/distributedlog/docs/latest/feed.xml">
<!-- Font Awesome -->
<script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script>
<!-- Google Analytics -->
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-83870961-1', 'auto');
ga('send', 'pageview');
</script>
<!-- End Google Analytics -->
<link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
</head>
<body role="document">
<nav class="navbar navbar-default navbar-fixed-top">
<div class="container">
<div class="navbar-header">
<a href="/" class="navbar-brand" >
<img alt="Brand" style="height: 28px" src="/docs/latest/images/distributedlog_logo_navbar.png">
</a>
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div id="navbar" class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<!-- Overview -->
<li><a href="/docs/latest/">V0.6.0</a></li>
<!-- Concepts -->
<li><a href="/docs/latest/basics/introduction">Concepts</a></li>
<!-- Quick Start -->
<li>
<a href="/docs/latest/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li>
<a href="/docs/latest/start/building.html">
Build DistributedLog from Source
</a>
</li>
<li>
<a href="/docs/latest/start/download.html">
Download Releases
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Quickstart</strong></li>
<li>
<a href="/docs/latest/start/quickstart.html">
Setup & Run Example
</a>
</li>
<li>
<a href="/docs/latest/tutorials/basic-1.html">
API - Write Records (via core library)
</a>
</li>
<li>
<a href="/docs/latest/tutorials/basic-2.html">
API - Write Records (via write proxy)
</a>
</li>
<li>
<a href="/docs/latest/tutorials/basic-5.html">
API - Read Records
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Deployment</strong></li>
<li>
<a href="/docs/latest/deployment/cluster.html">
Cluster Setup
</a>
</li>
<li>
<a href="/docs/latest/deployment/global-cluster.html">
Global Cluster Setup
</a>
</li>
<li>
<a href="/docs/latest/deployment/kubernetes.html">
Kubernetes
</a>
</li>
</ul>
</li>
<!-- API -->
<li>
<a href="/docs/latest/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="/docs/latest/api/java">Java</a></li>
</ul>
</li>
<!-- User Guide -->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
<a href="/docs/latest/basics/introduction.html">
Introduction
</a>
</li>
<li>
<a href="/docs/latest/user_guide/considerations/main.html">
Considerations
</a>
</li>
<li>
<a href="/docs/latest/user_guide/architecture/main.html">
Architecture
</a>
</li>
<li>
<a href="/docs/latest/user_guide/api/main.html">
API
</a>
</li>
<li>
<a href="/docs/latest/user_guide/configuration/main.html">
Configuration
</a>
</li>
<li>
<a href="/docs/latest/user_guide/design/main.html">
Detail Design
</a>
</li>
<li>
<a href="/docs/latest/user_guide/globalreplicatedlog/main.html">
Global Replicated Log
</a>
</li>
<li>
<a href="/docs/latest/user_guide/implementation/main.html">
Implementation
</a>
</li>
<li>
<a href="/docs/latest/user_guide/references/main.html">
References
</a>
</li>
</ul>
</li>
<!-- Admin Guide -->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="/docs/latest/deployment/cluster">Cluster Setup</a></li>
<li>
<a href="/docs/latest/admin_guide/operations.html">
Operations
</a>
</li>
<li>
<a href="/docs/latest/admin_guide/loadtest.html">
Load Test
</a>
</li>
<li>
<a href="/docs/latest/admin_guide/performance.html">
Performance Tuning
</a>
</li>
<li>
<a href="/docs/latest/admin_guide/hardware.html">
Hardware
</a>
</li>
<li>
<a href="/docs/latest/admin_guide/monitoring.html">
Monitoring
</a>
</li>
<li>
<a href="/docs/latest/admin_guide/zookeeper.html">
ZooKeeper
</a>
</li>
<li>
<a href="/docs/latest/admin_guide/bookkeeper.html">
BookKeeper
</a>
</li>
</ul>
</li>
<!-- Tutorials -->
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a>
<ul class="dropdown-menu">
<li class="dropdown-header"><strong>Basic</strong></li>
<li><a href="/docs/latest/tutorials/basic-1">Write Records (via Core Library)</a></li>
<li><a href="/docs/latest/tutorials/basic-2">Write Records (via Write Proxy)</a></li>
<li><a href="/docs/latest/tutorials/basic-3">Write Records to multiple streams</a></li>
<li><a href="/docs/latest/tutorials/basic-4">Atomic Write Records</a></li>
<li><a href="/docs/latest/tutorials/basic-5">Tailing Read Records</a></li>
<li><a href="/docs/latest/tutorials/basic-6">Rewind Read Records</a></li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Messaging</strong></li>
<li>
<a href="/docs/latest/tutorials/messaging-1.html">
Write records to partitioned streams
</a>
</li>
<li>
<a href="/docs/latest/tutorials/messaging-2.html">
Write records to multiple streams (load balancer)
</a>
</li>
<li>
<a href="/docs/latest/tutorials/messaging-3.html">
At-least-once Processing
</a>
</li>
<li>
<a href="/docs/latest/tutorials/messaging-4.html">
Exact-Once Processing
</a>
</li>
<li>
<a href="/docs/latest/tutorials/messaging-5.html">
Implement a kafka-like pub/sub system
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Replicated State Machines</strong></li>
<li>
<a href="/docs/latest/tutorials/replicatedstatemachines.html">
Build replicated state machines
</a>
</li>
<li role="separator" class="divider"></li>
<li class="dropdown-header"><strong>Analytics</strong></li>
<li><a href="/docs/latest/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li>
</ul>
</li>
</ul>
</div><!--/.nav-collapse -->
</div>
</nav>
<link rel="stylesheet" href="">
<div class="container" role="main">
<div class="row">
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<div class="row">
<!-- Sub Navigation -->
<div class="col-sm-3">
<ul id="sub-nav">
<li><a href="/docs/latest/user_guide/main.html" class="">User Guide</a>
<ul>
<li>
<a href="/docs/latest/basics/introduction.html" class="">
Introduction
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/considerations/main.html" class="">
Considerations
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/architecture/main.html" class="">
Architecture
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/api/main.html" class="">
API
</a>
<ul>
<li>
<a href="/docs/latest/user_guide/api/core.html" class="active">
Core Library API
</a>
</li>
<li>
<a href="/docs/latest/user_guide/api/proxy.html" class="active">
Proxy Client API
</a>
</li>
<li>
<a href="/docs/latest/user_guide/api/practice.html" class="active">
Best Practise
</a>
</li>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/configuration/main.html" class="">
Configuration
</a>
<ul>
<li>
<a href="/docs/latest/user_guide/configuration/core.html" class="active">
Core Library Configuration
</a>
</li>
<li>
<a href="/docs/latest/user_guide/configuration/proxy.html" class="active">
Write Proxy Configuration
</a>
</li>
<li>
<a href="/docs/latest/user_guide/configuration/client.html" class="active">
Client Configuration
</a>
</li>
<li>
<a href="/docs/latest/user_guide/configuration/perlog.html" class="active">
Per Stream Configuration
</a>
</li>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/design/main.html" class="">
Detail Design
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/globalreplicatedlog/main.html" class="">
Global Replicated Log
</a>
<ul>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/implementation/main.html" class="">
Implementation
</a>
<ul>
<li>
<a href="/docs/latest/user_guide/implementation/storage.html" class="active">
Storage
</a>
</li>
</ul>
</li>
<li>
<a href="/docs/latest/user_guide/references/main.html" class="">
References
</a>
<ul>
<li>
<a href="/docs/latest/user_guide/references/metrics.html" class="active">
Metrics
</a>
</li>
<li>
<a href="/docs/latest/user_guide/references/features.html" class="active">
Available Features
</a>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<!-- Main -->
<div class="col-sm-9">
<!-- Top anchor -->
<a href="#top"></a>
<!-- Breadcrumbs above the main heading -->
<ol class="breadcrumb">
<li><a href="/docs/latest/user_guide/main.html">User Guide</a></li>
<li><a href="/docs/latest/user_guide/configuration/main.html">Configuration</a></li>
<li class="active">Per Stream Configuration</li>
</ol>
<div class="text">
<!-- Content -->
<div class="contents topic" id="per-stream-configuration">
<p class="topic-title first">Per Stream Configuration</p>
<ul class="simple">
<li><a class="reference internal" href="#id1" id="id2">Per Stream Configuration</a><ul>
<li><a class="reference internal" href="#dynamic-configuration" id="id3">Dynamic Configuration</a><ul>
<li><a class="reference internal" href="#file-based-dynamic-configuration" id="id4">File Based Dynamic Configuration</a></li>
<li><a class="reference internal" href="#stream-config-provider" id="id5">Stream Config Provider</a></li>
<li><a class="reference internal" href="#available-dynamic-settings" id="id6">Available Dynamic Settings</a></li>
<li><a class="reference internal" href="#storage-settings" id="id7">Storage Settings</a></li>
<li><a class="reference internal" href="#transmit-settings" id="id8">Transmit Settings</a></li>
<li><a class="reference internal" href="#durability-settings" id="id9">Durability Settings</a></li>
<li><a class="reference internal" href="#readahead-settings" id="id10">ReadAhead Settings</a></li>
<li><a class="reference internal" href="#rate-limit-settings" id="id11">Rate Limit Settings</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="id1">
<h2><a class="toc-backref" href="#id2">Per Stream Configuration</a></h2>
<p>Application is allowed to override <cite>DistributedLogConfiguration</cite> for individual streams. This is archieved
for supplying an overrided <cite>DistributedLogConfiguration</cite> when opening the distributedlog manager.</p>
<pre class="literal-block">
DistributedLogNamespace namespace = ...;
DistributedLogConfiguration perStreamConf = new DistributeLogConfiguration();
perStreamConf.loadConf(...); // load configuration from a per stream configuration file
DistributedLogManager dlm = namespace.openLog(&quot;test-stream&quot;, Optional.of(perStreamConf), Optional.absent());
</pre>
<div class="section" id="dynamic-configuration">
<h3><a class="toc-backref" href="#id3">Dynamic Configuration</a></h3>
<p>Besides overriding normal <cite>DistributedLogConfiguration</cite> with per stream configuration, DistributedLog also
provides loading some configuration settings dynamically. The per stream dynamic settings are offered in
<cite>DynamicDistributedLogConfiguration</cite>.</p>
<div class="section" id="file-based-dynamic-configuration">
<h4><a class="toc-backref" href="#id4">File Based Dynamic Configuration</a></h4>
<p>The default dynamic configuration implementation is based on properties files and reloading the file periodically.</p>
<pre class="literal-block">
ConcurrentBaseConfiguration defaultConf = ...; // base config to fall through
int reloadPeriod = 60; // 60 seconds
TimeUnit reloadUnit = TimeUnit.SECOND;
String configPath = &quot;/path/to/per/stream/config/file&quot;;
File configFile = new File(configPath);
// load the fie into a properties configuration builder
PropertiesConfigurationBuilder properties =
new PropertiesConfigurationBuilder(configFile.toURI().toURL());
// Construct the dynamic configuration
DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(defaultConf);
// add a configuration subscription to periodically reload the config from the file
ConfigurationSubscription subscription =
new ConfigurationSubscription(dynConf, properties, executorService, reloadPeriod, reloadUnit);
</pre>
</div>
<div class="section" id="stream-config-provider">
<h4><a class="toc-backref" href="#id5">Stream Config Provider</a></h4>
<p>The stream config provider is designed to manage and reload dynamic configs for individual streams.</p>
<pre class="literal-block">
String perStreamConfigDir = &quot;/path/to/per/stream/config/dir&quot;;
String defaultConfigPath = &quot;/path/to/default/config/file&quot;;
StreamPartitionConverter converter = ...;
ScheduledExecutorService scheduler = ...;
int reloadPeriod = 60; // 60 seconds
TimeUnit reloadUnit = TimeUnit.SECOND;
StreamConfigProvider provider = new ServiceStreamConfigProvider(
perStreamConfigDir,
defaultConfigPath,
converter,
scheduler,
reloadPeriod,
reloadUnit);
Optional&lt;DynamicDistributedLogConfiguration&gt; streamConf = provider.getDynamicStreamConfig(&quot;test-stream&quot;);
</pre>
<ul class="simple">
<li><em>perStreamConfigDir</em>: The directory contains configuration files for each stream. the file name is <cite>&lt;stream_name&gt;.conf</cite>.</li>
<li><em>defaultConfigPath</em>: The default configuration file. If there is no stream configuration file found in <cite>perStreamConfigDir</cite>,
it would load the configuration from <cite>defaultConfigPath</cite>.</li>
<li><em>StreamPartitionConverter</em>: A converter that convert the stream names to partitions. DistributedLog doesn't provide built-in
partitions. It leaves partition strategy to application. Application usually put the partition id in the dl stream name. So the
converter is for group the streams and apply same configuration. For example, if application uses 3 streams and names them as
<cite>test-stream_000001</cite>, <cite>test-stream_000002</cite> and <cite>test-stream_000003</cite>, a <cite>StreamPartitionConverter</cite> could be used to categorize them
as partitions for stream <cite>test-stream</cite> and apply the configuration from file <cite>test-stream.conf</cite>.</li>
<li><em>scheduler</em>: The executor service that reloads configuration periodically.</li>
<li><em>reloadPeriod</em>: The reload period, in <cite>reloadUnit</cite>.</li>
</ul>
</div>
<div class="section" id="available-dynamic-settings">
<h4><a class="toc-backref" href="#id6">Available Dynamic Settings</a></h4>
</div>
<div class="section" id="storage-settings">
<h4><a class="toc-backref" href="#id7">Storage Settings</a></h4>
<ul class="simple">
<li><em>logSegmentRetentionHours</em>: The log segment retention period, in hours. In other words, how long should DL keep the log segment once it is <cite>truncated</cite> or <cite>completed</cite>.</li>
<li><em>bkcEnsembleSize</em>: The ensemble size of the log segment. The default value is 3.</li>
<li><em>bkcWriteQuorumSize</em>: The write quorum size of the log segment. The default value is 3.</li>
<li><em>bkcAckQuorumSize</em>: The ack quorumm size of the log segment. The default value is 2.</li>
</ul>
</div>
<div class="section" id="transmit-settings">
<h4><a class="toc-backref" href="#id8">Transmit Settings</a></h4>
<ul class="simple">
<li><em>writerOutputBufferSize</em>: The output buffer size in bytes. Larger buffer size will result in higher compression ratio and
it would reduce the entries sent to bookkeeper, use the disk bandwidth more efficiently and improve throughput.
Set this setting to <cite>0</cite> will ask DL to transmit the data immediately, which it would achieve low latency.</li>
</ul>
</div>
<div class="section" id="durability-settings">
<h4><a class="toc-backref" href="#id9">Durability Settings</a></h4>
<ul class="simple">
<li><em>isDurableWriteEnabled</em>: The flag indicates whether durable write is enabled. By default it is true.</li>
</ul>
</div>
<div class="section" id="readahead-settings">
<h4><a class="toc-backref" href="#id10">ReadAhead Settings</a></h4>
<ul class="simple">
<li><em>readAheadMaxRecords</em>: The maximum number of records that will be cached in readahead cache by the DL readers. The default value
is 10. A higher value will improve throughput but use more memory. It should be tuned properly to avoid jvm gc if the reader cannot
keep up with the writing rate.</li>
<li><em>readAheadBatchSize</em>: The maximum number of entries that readahead worker will read in one batch. The default value is 2.
Increase the value to increase the concurrency of reading entries from bookkeeper. It is recommended to tune to a proper value for
catching up readers, not to exhaust bookkeeper's bandwidth.</li>
</ul>
</div>
<div class="section" id="rate-limit-settings">
<h4><a class="toc-backref" href="#id11">Rate Limit Settings</a></h4>
<p>All the rate limit settings have both <cite>soft</cite> and <cite>hard</cite> thresholds. If the throughput goes above <cite>soft</cite> limit,
the requests won't be rejected but just logging in the stat. But if the throughput goes above <cite>hard</cite> limit,
the requests would be rejected immediately.</p>
<p>NOTE: <cite>bps</cite> stands for <cite>bytes per second</cite>, while <cite>rps</cite> stands for <cite>requests per second</cite>.</p>
<ul class="simple">
<li><em>bpsSoftWriteLimit</em>: The soft limit for bps. Setting it to 0 or negative value will disable this feature.
By default it is disabled.</li>
<li><em>bpsHardWriteLimit</em>: The hard limit for bps. Setting it to 0 or negative value will disable this feature.
By default it is disabled.</li>
<li><em>rpsSoftWriteLimit</em>: The soft limit for rps. Setting it to 0 or negative value will disable this feature.
By default it is disabled.</li>
<li><em>rpsHardWriteLimit</em>: The hard limit for rps. Setting it to 0 or negative value will disable this feature.
By default it is disabled.</li>
</ul>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<hr>
<div class="row">
<div class="col-xs-12">
<footer>
<p class="text-center">&copy; Copyright 2016
<a href="http://www.apache.org">The Apache Software Foundation.</a> All Rights Reserved.
</p>
<p class="text-center">
<a href="/docs/latest/feed.xml">RSS Feed</a>
</p>
</footer>
</div>
</div>
<!-- container div end -->
</div>
<script>
(function () {
'use strict';
anchors.options.placement = 'right';
anchors.add();
})();
</script>
</body>
</html>