Google Git
Sign in
apache / distributedlog / refs/heads/gh-pages / . / references / metrics.html
blob: 75266ec1bda98b4f0a2c809d06ea950822e88617 [file] [log] [blame]
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Metrics &mdash; DistributedLog 1.0 documentation</title>
<link rel="stylesheet" href="../_static/override.css" type="text/css" />
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/bootstrap-3.1.0/css/bootstrap.min.css" type="text/css" />
<link rel="stylesheet" href="../_static/bootstrap-3.1.0/css/bootstrap-theme.min.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/featherlight.min.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/docbird.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/docbird-xs.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/jquery.rateyo.min.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/selection-sharer.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../',
VERSION: '1.0',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
<script type="text/javascript" src="../_static/underscore.js"></script>
<script type="text/javascript" src="../_static/doctools.js"></script>
<script type="text/javascript" src="../_static/bootstrap-3.1.0/js/bootstrap.min.js"></script>
<script type="text/javascript" src="../_static/js/bootstrap-docbird.js"></script>
<script type="text/javascript" src="../_static/js/jquery-1.11.0.min.js"></script>
<script type="text/javascript" src="../_static/js/jquery-fix.js"></script>
<script type="text/javascript" src="../_static/js/featherlight.min.js"></script>
<script type="text/javascript" src="../_static/js/ifvisible.js"></script>
<script type="text/javascript" src="../_static/js/timeme.js"></script>
<script type="text/javascript" src="../_static/js/jquery.rateyo.min.js"></script>
<script type="text/javascript" src="../_static/js/js.cookie.js"></script>
<link rel="shortcut icon" href="../_static/docbird.ico"/>
<link rel="top" title="DistributedLog 1.0 documentation" href="../index.html" />
<link rel="up" title="References" href="main.html" />
<link rel="next" title="Features" href="features.html" />
<link rel="prev" title="Configuration Settings" href="configuration.html" />
<meta charset='utf-8'>
<meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
<meta name="apple-mobile-web-app-capable" content="yes">
<meta property="docbird:project" content="DistributedLog" />
</head>
<body>
<div class="navbar navbar-default navbar-fixed-top" role="navigation">
<div class="container-fluid">
<div class="row db-header">
<div class="col-sm-3 col-md-3 col-lg-3 hidden-xs db-header-controls">
<a href="/" alt="Back to Docbird">
<div class="db-home-button">
<span class="glyphicon glyphicon-home"></span>
</div>
</a>
<form action="../search.html" method="get" class="db-searchbox-form">
<div class="form-group">
<input type="text" name="q" class="form-control db-searchbox-input" placeholder="Search DistributedLog" />
</div>
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
<div class="col-sm-7 col-md-7 col-lg-7 col-xs-12 db-header-info">
<div class="visible-xs">
<a href="/" alt="Back to Docbird">
<div class="db-home-button">
<span class="glyphicon glyphicon-home"></span>
</div>
</a>
</div>
<div class="visible-xs db-xs-menu-button">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#db-xs-menu">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
</div>
<div class="db-header-projectname">
<h1><a href="../index.html">DistributedLog</a></h1>
</div>
</div>
</div>
<div class="row db-xs-menu hidden-sm hidden-md hidden-lg
collapse" id="db-xs-menu">
<form action="../search.html" method="get" class="db-searchbox-form">
<div class="form-group">
<input type="text" name="q" class="form-control db-searchbox-input" placeholder="Search DistributedLog" />
</div>
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
<div class="db-toc" role="complementary">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../download.html">Releases</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../download.html#rc1">0.3.51-RC1</a></li>
<li class="toctree-l2"><a class="reference internal" href="../download.html#rc0">0.3.51-RC0</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../basics/main.html">Getting Started</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../basics/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../basics/quickstart.html">Quick Start</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../api/main.html">API</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../api/core.html">Core Library API</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/proxy.html">Write Proxy Client API</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/practice.html">Best Practices</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../configuration/main.html">Configuration</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../configuration/core.html">Core Library Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../configuration/proxy.html">Write Proxy Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../configuration/client.html">Client Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../configuration/perlog.html">Per Stream Configuration</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../considerations/main.html">Considerations</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#consistency-durability-and-ordering">Consistency, Durability and Ordering</a></li>
<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#partitioning">Partitioning</a></li>
<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#processing-semantics">Processing Semantics</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../architecture/main.html">Architecture</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#data-model">Data Model</a></li>
<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#software-stack">Software Stack</a></li>
<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#lifecyle-of-records">Lifecyle of records</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../design/main.html">Detail Design</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../design/main.html#consistency">Consistency</a></li>
<li class="toctree-l2"><a class="reference internal" href="../design/main.html#streaming-reads">Streaming Reads</a></li>
<li class="toctree-l2"><a class="reference internal" href="../design/main.html#logsegment-lifecycle">LogSegment Lifecycle</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../globalreplicatedlog/main.html">Global Replicated Log</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#region-aware-data-placement-policy">Region Aware Data Placement Policy</a></li>
<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#cross-region-speculative-reads">Cross Region Speculative Reads</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../implementation/main.html">Implementation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../implementation/storage.html">Storage</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../operations/main.html">Deployment &amp; Administration</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../operations/deployment.html">Cluster Setup &amp; Deployment</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/operations.html">DistributedLog Operations</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/performance.html">Performance Tuning</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/hardware.html">Hardware</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/monitoring.html">Monitoring</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/zookeeper.html">ZooKeeper</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/bookkeeper.html">BookKeeper</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../performance/main.html">Performance</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="main.html">References</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="configuration.html">Configuration Settings</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="">Metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="features.html">Features</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/main.html">Tutorials</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#basic">Basic</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#messaging">Messaging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#replicated-state-machines">Replicated State Machines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#analytics">Analytics</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../developer/main.html">Developer</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../developer/release.html">Release</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
</ul>
</div>
</div>
</div>
</div>
<div class="container">
<div class="row">
<div style="z-index: 1" class="col-xs-12 col-sm-12 col-md-12 col-lg-12">
<style>
.overflow-container {
display: none;
}
.overflow-toggle {
text-decoration: none;
border-bottom: none;
border-radius: 4px;
border: 1px solid #eee;
padding: 1px 3px 3px;
color: #888;
font-weight: normal;
background-color: linen;
line-height: 1.85em;
cursor: pointer;
}
.overflow-toggle:hover {
color: #333;
border-color: #ccc;
background-color: beige;
}
</style>
<script>
$(function(){
$('.overflow-toggle').on('click', function(){
$(this).next().toggle();
});
});
</script>
<div class="db-project-header-container">
<div class="row">
<div class="db-project-info col-lg-12 col-md-12 col-sm-12 col-xs-12">
<h1>
<a href="../index.html">DistributedLog</a>
</h1>
<div class="db-code-link">
<a href="git@github.com:twitter/distributedlog.git/tree/master/" target="_blank">git@github.com:twitter/distributedlog.git/tree/master/</a>
</div>
</div>
</div>
<div class="row db-project-links-row">
<div class=" col-sm-3 col-md-3 col-lg-3 db-project-link-column">
<div class="db-hashtag-container">
<span class="db-project-link-label">OWNERS</span>
<em>None</em>
</div>
</div>
<div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column">
<div class="db-hashtag-container">
<span class="db-project-link-label">TAGS</span>
<em><a class="db-hashtag" href="/?q=tags:%23uses_maven">#uses_maven</a></em>
</div>
</div>
<div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column">
<span class="db-project-link-label">HEALTH</span>
<h3 style="margin-top: 0">
<!-- <a href="/techdocs/checklist.html" class="label label-success">-->
<a href="/report/distributedlog" class="">
9.0 / 10
<span style="margin-left: .25em" class="glyphicon glyphicon-ok"></span>
</a>
</h3>
</div>
<div class="col-sm-3 col-md-3 col-lg-3 db-project-link-column">
<span class="db-project-link-label">RATING</span>
<div id="rateYo"></div>
</div>
</div>
</div>
</div>
<div class="col-xs-12 col-sm-8 col-md-8 col-lg-8">
<div class="db-content-body">
<div class="section" id="metrics">
<h1>Metrics<a class="headerlink" href="#metrics" title="Permalink to this headline">¶</a></h1>
<p>This section lists the metrics exposed by main classes.</p>
<p>({scope} is referencing current scope value of passed in StatsLogger.)</p>
<div class="section" id="monitoredfuturepool">
<h2>MonitoredFuturePool<a class="headerlink" href="#monitoredfuturepool" title="Permalink to this headline">¶</a></h2>
<p><strong>{scope}/tasks_pending</strong></p>
<p>Gauge. How many tasks are pending in this future pool? If this value becomes high, it means that
the future pool execution rate couldn't keep up with submission rate. That would be cause high
<em>task_pending_time</em> hence affecting the callers that use this future pool.
It could also cause heavy jvm gc if this pool keeps building up.</p>
<p><strong>{scope}/task_pending_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on waiting being executed.
It becomes high because either <em>tasks_pending</em> is building up or <em>task_execution_time</em> is high blocking other
tasks to execute.</p>
<p><strong>{scope}/task_execution_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on execution. If it becomes high,
it would block other tasks to execute if there isn't enough threads in this executor, hence cause high
<em>task_pending_time</em> and impact user end latency.</p>
<p><strong>{scope}/task_enqueue_time</strong></p>
<p>OpStats. The time that tasks spent on submission. The submission time would also impact user end latency.</p>
</div>
<div class="section" id="monitoredscheduledthreadpoolexecutor">
<h2>MonitoredScheduledThreadPoolExecutor<a class="headerlink" href="#monitoredscheduledthreadpoolexecutor" title="Permalink to this headline">¶</a></h2>
<p><strong>{scope}/pending_tasks</strong></p>
<p>Gauge. How many tasks are pending in this thread pool executor? If this value becomes high, it means that
the thread pool executor execution rate couldn't keep up with submission rate. That would be cause high
<em>task_pending_time</em> hence affecting the callers that use this executor. It could also cause heavy jvm gc if
queue keeps building up.</p>
<p><strong>{scope}/completed_tasks</strong></p>
<p>Gauge. How many tasks are completed in this thread pool executor?</p>
<p><strong>{scope}/total_tasks</strong></p>
<p>Gauge. How many tasks are submitted to this thread pool executor?</p>
<p><strong>{scope}/task_pending_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on waiting being executed.
It becomes high because either <em>pending_tasks</em> is building up or <em>task_execution_time</em> is high blocking other
tasks to execute.</p>
<p><strong>{scope}/task_execution_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on execution. If it becomes high,
it would block other tasks to execute if there isn't enough threads in this executor, hence cause high
<em>task_pending_time</em> and impact user end latency.</p>
</div>
<div class="section" id="orderedscheduler">
<h2>OrderedScheduler<a class="headerlink" href="#orderedscheduler" title="Permalink to this headline">¶</a></h2>
<p>OrderedScheduler is a thread pool based <em>ScheduledExecutorService</em>. It is comprised with multiple
<a class="reference internal" href="#monitoredscheduledthreadpoolexecutor">MonitoredScheduledThreadPoolExecutor</a>. Each <a class="reference internal" href="#monitoredscheduledthreadpoolexecutor">MonitoredScheduledThreadPoolExecutor</a> is wrapped into a
<a class="reference internal" href="#monitoredfuturepool">MonitoredFuturePool</a>. So there are aggregated stats and per-executor stats exposed.</p>
<div class="section" id="aggregated-stats">
<h3>Aggregated Stats<a class="headerlink" href="#aggregated-stats" title="Permalink to this headline">¶</a></h3>
<p><strong>{scope}/task_pending_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on waiting being executed.
It becomes high because either <em>pending_tasks</em> is building up or <em>task_execution_time</em> is high blocking other
tasks to execute.</p>
<p><strong>{scope}/task_execution_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on execution. If it becomes high,
it would block other tasks to execute if there isn't enough threads in this executor, hence cause high
<em>task_pending_time</em> and impact user end latency.</p>
<p><strong>{scope}/futurepool/tasks_pending</strong></p>
<p>Gauge. How many tasks are pending in this future pool? If this value becomes high, it means that
the future pool execution rate couldn't keep up with submission rate. That would be cause high
<em>task_pending_time</em> hence affecting the callers that use this future pool.
It could also cause heavy jvm gc if this pool keeps building up.</p>
<p><strong>{scope}/futurepool/task_pending_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on waiting being executed.
It becomes high because either <em>tasks_pending</em> is building up or <em>task_execution_time</em> is high blocking other
tasks to execute.</p>
<p><strong>{scope}/futurepool/task_execution_time</strong></p>
<p>OpStats. It measures the characteristics about the time that tasks spent on execution. If it becomes high,
it would block other tasks to execute if there isn't enough threads in this executor, hence cause high
<em>task_pending_time</em> and impact user end latency.</p>
<p><strong>{scope}/futurepool/task_enqueue_time</strong></p>
<p>OpStats. The time that tasks spent on submission. The submission time would also impact user end latency.</p>
</div>
<div class="section" id="per-executor-stats">
<h3>Per Executor Stats<a class="headerlink" href="#per-executor-stats" title="Permalink to this headline">¶</a></h3>
<p>Stats about individual executors are exposed under <em>{scope}/{name}-executor-{id}-0</em>. <em>{name}</em> is the scheduler
name and <em>{id}</em> is the index of the executor in the pool. The corresponding stats of its futurepool are exposed
under <em>{scope}/{name}-executor-{id}-0/futurepool</em>. See <a class="reference internal" href="#monitoredscheduledthreadpoolexecutor">MonitoredScheduledThreadPoolExecutor</a> and <a class="reference internal" href="#monitoredfuturepool">MonitoredFuturePool</a>
for more details.</p>
</div>
</div>
<div class="section" id="zookeeperclient">
<h2>ZooKeeperClient<a class="headerlink" href="#zookeeperclient" title="Permalink to this headline">¶</a></h2>
<div class="section" id="operation-stats">
<h3>Operation Stats<a class="headerlink" href="#operation-stats" title="Permalink to this headline">¶</a></h3>
<p>All operation stats are exposed under {scope}/zk. The stats are <strong>latency</strong> <em>OpStats</em>
on zookeeper operations.</p>
<p><strong>{scope}/zk/{op}</strong></p>
<p>latency stats on operations.
these operations are <em>create_client</em>, <em>get_data</em>, <em>set_data</em>, <em>delete</em>, <em>get_children</em>, <em>multi</em>, <em>get_acl</em>, <em>set_acl</em> and <em>sync</em>.</p>
</div>
<div class="section" id="watched-event-stats">
<h3>Watched Event Stats<a class="headerlink" href="#watched-event-stats" title="Permalink to this headline">¶</a></h3>
<p>All stats on zookeeper watched events are exposed under {scope}/watcher. The stats are <em>Counter</em>
about the watched events that this client received:</p>
<p><strong>{scope}/watcher/state/{keeper_state}</strong></p>
<p>the number of <cite>KeeperState</cite> changes that this client received. The states are <em>Disconnected</em>, <em>SyncConnected</em>,
<em>AuthFailed</em>, <em>ConnectedReadOnly</em>, <em>SaslAuthenticated</em> and <em>Expired</em>. By monitoring metrics like <em>SyncConnected</em>
or <em>Expired</em> it would help understanding the healthy of this zookeeper client.</p>
<p><strong>{scope}/watcher/events/{event}</strong></p>
<p>the number of <a href="#id1"><span class="problematic" id="id2">`</span></a>Watcher.Event`s received by this client. Those events are <em>None</em>, <em>NodeCreated</em>, <em>NodeDeleted</em>,
<em>NodeDataChanged</em>, <em>NodeChildrenChanged</em>.</p>
</div>
<div class="section" id="watcher-manager-stats">
<h3>Watcher Manager Stats<a class="headerlink" href="#watcher-manager-stats" title="Permalink to this headline">¶</a></h3>
<p>This ZooKeeperClient provides a watcher manager to manage watchers for applications. It tracks the mapping between
paths and watcher. It is the way to provide the ability on removing watchers. The stats are <em>Gauge</em> about the number
of watchers managed by this zookeeper client.</p>
<p><strong>{scope}/watcher_manager/total_watches</strong></p>
<p>total number of watches that are managed by this watcher manager. If it keeps growing, it usually means that
watchers are leaking (resources aren't closed properly). It will cause OOM.</p>
<p><strong>{scope}/watcher_manager/num_child_watches</strong></p>
<p>total number of paths that are watched by this watcher manager.</p>
</div>
</div>
<div class="section" id="bookkeeperclient">
<h2>BookKeeperClient<a class="headerlink" href="#bookkeeperclient" title="Permalink to this headline">¶</a></h2>
<p>TODO: add bookkeeper stats there</p>
</div>
<div class="section" id="distributedreentrantlock">
<h2>DistributedReentrantLock<a class="headerlink" href="#distributedreentrantlock" title="Permalink to this headline">¶</a></h2>
<p>All stats related to locks are exposed under {scope}/lock.</p>
<p><strong>{scope}/acquire</strong></p>
<p>OpStats. It measures the characteristics about the time that spent on acquiring locks.</p>
<p><strong>{scope}/release</strong></p>
<p>OpStats. It measures the characteristics about the time that spent on releasing locks.</p>
<p><strong>{scope}/reacquire</strong></p>
<p>OpStats. The lock will be expired when the underneath zookeeper session expired. The
reentrant lock will attempt to re-acquire the lock automatically when session expired.
This metric measures the characteristics about the time that spent on re-acquiring locks.</p>
<p><strong>{scope}/internalTryRetries</strong></p>
<p>Counter. The number of retries that locks spend on re-creating internal locks. Typically,
a new internal lock will be created when session expired.</p>
<p><strong>{scope}/acquireTimeouts</strong></p>
<p>Counter. The number of timeouts that caller experienced when acquiring locks.</p>
<p><strong>{scope}/tryAcquire</strong></p>
<p>OpStats. It measures the characteristics about the time that each internal lock spent on
acquiring.</p>
<p><strong>{scope}/tryTimeouts</strong></p>
<p>Counter. The number of timeouts that internal locks try acquiring.</p>
<p><strong>{scope}/unlock</strong></p>
<p>OpStats. It measures the characteristics about the time that the caller spent on unlocking
internal locks.</p>
</div>
<div class="section" id="bkloghandler">
<h2>BKLogHandler<a class="headerlink" href="#bkloghandler" title="Permalink to this headline">¶</a></h2>
<p>The log handler is a base class on managing log segments. so all the metrics in this class are
related log segments retrieval and exposed under {scope}/logsegments. They are all <cite>OpStats</cite> in
the format of <cite>{scope}/logsegments/{op}</cite>. Those operations are:</p>
<ul class="simple">
<li>force_get_list: force to get the list of log segments.</li>
<li>get_list: get the list of the log segments. it might just retrieve from local log segment cache.</li>
<li>get_filtered_list: get the filtered list of log segments.</li>
<li>get_full_list: get the full list of log segments.</li>
<li>get_inprogress_segment: time between the inprogress log segment created and the handler read it.</li>
<li>get_completed_segment: time between a log segment is turned to completed and the handler read it.</li>
<li>negative_get_inprogress_segment: record the negative values for <cite>get_inprogress_segment</cite>.</li>
<li>negative_get_completed_segment: record the negative values for <cite>get_completed_segment</cite>.</li>
<li>recover_last_entry: recovering last entry from a log segment.</li>
<li>recover_scanned_entries: the number of entries that are scanned during recovering.</li>
</ul>
<p>See <a class="reference internal" href="#bklogwritehandler">BKLogWriteHandler</a> for write handlers.</p>
<p>See <a class="reference internal" href="#bklogreadhandler">BKLogReadHandler</a> for read handlers.</p>
</div>
<div class="section" id="bklogreadhandler">
<h2>BKLogReadHandler<a class="headerlink" href="#bklogreadhandler" title="Permalink to this headline">¶</a></h2>
<p>The core logic in log reader handle is readahead worker. Most of readahead stats are exposed under
{scope}/readahead_worker.</p>
<p><strong>{scope}/readahead_worker/wait</strong></p>
<p>Counter. Number of waits that readahead worker is waiting. If this keeps increasing, it usually means
readahead keep getting full because of reader slows down reading.</p>
<p><strong>{scope}/readahead_worker/repositions</strong></p>
<p>Counter. Number of repositions that readhead worker encounters. Reposition means that a readahead worker
finds that it isn't advancing to a new log segment and force re-positioning.</p>
<p><strong>{scope}/readahead_worker/entry_piggy_back_hits</strong></p>
<p>Counter. It increases when the last add confirmed being advanced because of the piggy-back lac.</p>
<p><strong>{scope}/readahead_worker/entry_piggy_back_misses</strong></p>
<p>Counter. It increases when the last add confirmed isn't advanced by a read entry because it doesn't
iggy back a newer lac.</p>
<p><strong>{scope}/readahead_worker/read_entries</strong></p>
<p>OpStats. Stats on number of entries read per readahead read batch.</p>
<p><strong>{scope}/readahead_worker/read_lac_counter</strong></p>
<p>Counter. Stats on the number of readLastConfirmed operations</p>
<p><strong>{scope}/readahead_worker/read_lac_and_entry_counter</strong></p>
<p>Counter. Stats on the number of readLastConfirmedAndEntry operations.</p>
<p><strong>{scope}/readahead_worker/cache_full</strong></p>
<p>Counter. It increases each time readahead worker finds cache become full. If it keeps increasing,
that means reader slows down reading.</p>
<p><strong>{scope}/readahead_worker/resume</strong></p>
<p>OpStats. Stats on readahead worker resuming reading from wait state.</p>
<p><strong>{scope}/readahead_worker/long_poll_interruption</strong></p>
<p>OpStats. Stats on the number of interruptions happened to long poll. the interruptions are usually
because of receiving zookeeper notifications.</p>
<p><strong>{scope}/readahead_worker/notification_execution</strong></p>
<p>OpStats. Stats on executions over the notifications received from zookeeper.</p>
<p><strong>{scope}/readahead_worker/metadata_reinitialization</strong></p>
<p>OpStats. Stats on metadata reinitialization after receiving notifcation from log segments updates.</p>
<p><strong>{scope}/readahead_worker/idle_reader_warn</strong></p>
<p>Counter. It increases each time the readahead worker detects itself becoming idle.</p>
</div>
<div class="section" id="bklogwritehandler">
<h2>BKLogWriteHandler<a class="headerlink" href="#bklogwritehandler" title="Permalink to this headline">¶</a></h2>
<p>Log write handlers are responsible for log segment creation/deletions. All the metrics are exposed under
{scope}/segments.</p>
<p><strong>{scope}/segments/open</strong></p>
<p>OpStats. Latency characteristics on starting a new log segment.</p>
<p><strong>{scope}/segments/close</strong></p>
<p>OpStats. Latency characteristics on completing an inprogress log segment.</p>
<p><strong>{scope}/segments/recover</strong></p>
<p>OpStats. Latency characteristics on recovering a log segment.</p>
<p><strong>{scope}/segments/delete</strong></p>
<p>OpStats. Latency characteristics on deleting a log segment.</p>
</div>
<div class="section" id="bkasynclogwriter">
<h2>BKAsyncLogWriter<a class="headerlink" href="#bkasynclogwriter" title="Permalink to this headline">¶</a></h2>
<p><strong>{scope}/log_writer/write</strong></p>
<p>OpStats. latency characteristics about the time that write operations spent.</p>
<p><strong>{scope}/log_writer/write/queued</strong></p>
<p>OpStats. latency characteristics about the time that write operations spent in the queue.
<cite>{scope}/log_writer/write</cite> latency is high might because the write operations are pending
in the queue for long time due to log segment rolling.</p>
<p><strong>{scope}/log_writer/bulk_write</strong></p>
<p>OpStats. latency characteristics about the time that bulk_write operations spent.</p>
<p><strong>{scope}/log_writer/bulk_write/queued</strong></p>
<p>OpStats. latency characteristics about the time that bulk_write operations spent in the queue.
<cite>{scope}/log_writer/bulk_write</cite> latency is high might because the write operations are pending
in the queue for long time due to log segment rolling.</p>
<p><strong>{scope}/log_writer/get_writer</strong></p>
<p>OpStats. the time spent on getting the writer. it could spike when there is log segment rolling
happened during getting the writer. it is a good stat to look into when the latency is caused by
queuing time.</p>
<p><strong>{scope}/log_writer/pending_request_dispatch</strong></p>
<p>Counter. the number of queued operations that are dispatched after log segment is rolled. it is
an metric on measuring how many operations has been queued because of log segment rolling.</p>
</div>
<div class="section" id="bkasynclogreader">
<h2>BKAsyncLogReader<a class="headerlink" href="#bkasynclogreader" title="Permalink to this headline">¶</a></h2>
<p><strong>{scope}/async_reader/future_set</strong></p>
<p>OpStats. Time spent on satisfying futures of read requests. if it is high, it means that the caller
takes time on processing the result of read requests. The side effect is blocking consequent reads.</p>
<p><strong>{scope}/async_reader/schedule</strong></p>
<p>OpStats. Time spent on scheduling next reads.</p>
<p><strong>{scope}/async_reader/background_read</strong></p>
<p>OpStats. Time spent on background reads.</p>
<p><strong>{scope}/async_reader/read_next_exec</strong></p>
<p>OpStats. Time spent on executing <cite>reader#readNext()</cite></p>
<p><strong>{scope}/async_reader/time_between_read_next</strong></p>
<p>OpStats. Time spent on between two consequent <cite>reader#readNext()</cite>. if it is high, it means that
the caller is slowing down on calling <cite>reader#readNext()</cite>.</p>
<p><strong>{scope}/async_reader/delay_until_promise_satisfied</strong></p>
<p>OpStats. Total latency for the read requests.</p>
<p><strong>{scope}/async_reader/idle_reader_error</strong></p>
<p>Counter. The number idle reader errors.</p>
</div>
<div class="section" id="bkdistributedlogmanager">
<h2>BKDistributedLogManager<a class="headerlink" href="#bkdistributedlogmanager" title="Permalink to this headline">¶</a></h2>
<div class="section" id="future-pools">
<h3>Future Pools<a class="headerlink" href="#future-pools" title="Permalink to this headline">¶</a></h3>
<p>The stats about future pools that used by writers are exposed under {scope}/writer_future_pool,
while the stats about future pools that used by readers are exposed under {scope}/reader_future_pool.
See <a class="reference internal" href="#monitoredfuturepool">MonitoredFuturePool</a> for detail stats.</p>
</div>
<div class="section" id="distributed-locks">
<h3>Distributed Locks<a class="headerlink" href="#distributed-locks" title="Permalink to this headline">¶</a></h3>
<p>The stats about the locks used by writers are exposed under {scope}/lock while those used by readers
are exposed under {scope}/read_lock/lock. See <a class="reference internal" href="#distributedreentrantlock">DistributedReentrantLock</a> for detail stats.</p>
</div>
<div class="section" id="log-handlers">
<h3>Log Handlers<a class="headerlink" href="#log-handlers" title="Permalink to this headline">¶</a></h3>
<p><strong>{scope}/logsegments</strong></p>
<p>All basic stats of log handlers are exposed under {scope}/logsegments. See <a class="reference internal" href="#bkloghandler">BKLogHandler</a> for detail stats.</p>
<p><strong>{scope}/segments</strong></p>
<p>The stats about write log handlers are exposed under {scope}/segments. See <a class="reference internal" href="#bklogwritehandler">BKLogWriteHandler</a> for detail stats.</p>
<p><strong>{scope}/readhead_worker</strong></p>
<p>The stats about read log handlers are exposed under {scope}/readahead_worker.
See <a class="reference internal" href="#bklogreadhandler">BKLogReadHandler</a> for detail stats.</p>
</div>
<div class="section" id="writers">
<h3>Writers<a class="headerlink" href="#writers" title="Permalink to this headline">¶</a></h3>
<p>All writer related metrics are exposed under {scope}/log_writer. See <a class="reference internal" href="#bkasynclogwriter">BKAsyncLogWriter</a> for detail stats.</p>
</div>
<div class="section" id="readers">
<h3>Readers<a class="headerlink" href="#readers" title="Permalink to this headline">¶</a></h3>
<p>All reader related metrics are exposed under {scope}/async_reader. See <a class="reference internal" href="#bkasynclogreader">BKAsyncLogReader</a> for detail stats.</p>
</div>
</div>
<div class="section" id="bkdistributedlognamespace">
<h2>BKDistributedLogNamespace<a class="headerlink" href="#bkdistributedlognamespace" title="Permalink to this headline">¶</a></h2>
<div class="section" id="zookeeper-clients">
<h3>ZooKeeper Clients<a class="headerlink" href="#zookeeper-clients" title="Permalink to this headline">¶</a></h3>
<p>There are various of zookeeper clients created per namespace for different purposes. They are:</p>
<p><strong>{scope}/dlzk_factory_writer_shared</strong></p>
<p>Stats about the zookeeper client shared by all DL writers.</p>
<p><strong>{scope}/dlzk_factory_reader_shared</strong></p>
<p>Stats about the zookeeper client shared by all DL readers.</p>
<p><strong>{scope}/bkzk_factory_writer_shared</strong></p>
<p>Stats about the zookeeper client used by bookkeeper client that shared by all DL writers.</p>
<p><strong>{scope}/bkzk_factory_reader_shared</strong></p>
<p>Stats about the zookeeper client used by bookkeeper client that shared by all DL readers.</p>
<p>See <a class="reference internal" href="#zookeeperclient">ZooKeeperClient</a> for zookeeper detail stats.</p>
</div>
<div class="section" id="bookkeeper-clients">
<h3>BookKeeper Clients<a class="headerlink" href="#bookkeeper-clients" title="Permalink to this headline">¶</a></h3>
<p>All the bookkeeper client related stats are exposed directly to current {scope}. See <a class="reference internal" href="#bookkeeperclient">BookKeeperClient</a>
for detail stats.</p>
</div>
<div class="section" id="utils">
<h3>Utils<a class="headerlink" href="#utils" title="Permalink to this headline">¶</a></h3>
<p><strong>{scope}/factory/thread_pool</strong></p>
<p>Stats about the ordered scheduler used by this namespace. See <a class="reference internal" href="#orderedscheduler">OrderedScheduler</a> for detail stats.</p>
<p><strong>{scope}/factory/readahead_thread_pool</strong></p>
<p>Stats about the readahead thread pool executor used by this namespace. See <a class="reference internal" href="#monitoredscheduledthreadpoolexecutor">MonitoredScheduledThreadPoolExecutor</a>
for detail stats.</p>
<p><strong>{scope}/writeLimiter</strong></p>
<p>Stats about the global write limiter used by list namespace.</p>
</div>
<div class="section" id="distributedlogmanager">
<h3>DistributedLogManager<a class="headerlink" href="#distributedlogmanager" title="Permalink to this headline">¶</a></h3>
<p>All the core stats about reader and writer are exposed under current {scope} via <a class="reference internal" href="#bkdistributedlogmanager">BKDistributedLogManager</a>.</p>
</div>
</div>
</div>
</div>
</div>
<div class="hidden-xs col-sm-3 col-md-3 col-md-offset-1 col-lg-3 db-sidebar">
<div class="db-toc" role="complementary">
<ul class="current">
<li class="toctree-l0 current"><a class="current reference internal" href="../index.html">DistributedLog</a></li>
</ul>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../download.html">Releases</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../download.html#rc1">0.3.51-RC1</a></li>
<li class="toctree-l2"><a class="reference internal" href="../download.html#rc0">0.3.51-RC0</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../basics/main.html">Getting Started</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../basics/introduction.html">Introduction</a></li>
<li class="toctree-l2"><a class="reference internal" href="../basics/quickstart.html">Quick Start</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../api/main.html">API</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../api/core.html">Core Library API</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/proxy.html">Write Proxy Client API</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/practice.html">Best Practices</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../configuration/main.html">Configuration</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../configuration/core.html">Core Library Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../configuration/proxy.html">Write Proxy Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../configuration/client.html">Client Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../configuration/perlog.html">Per Stream Configuration</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../considerations/main.html">Considerations</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#consistency-durability-and-ordering">Consistency, Durability and Ordering</a></li>
<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#partitioning">Partitioning</a></li>
<li class="toctree-l2"><a class="reference internal" href="../considerations/main.html#processing-semantics">Processing Semantics</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../architecture/main.html">Architecture</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#data-model">Data Model</a></li>
<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#software-stack">Software Stack</a></li>
<li class="toctree-l2"><a class="reference internal" href="../architecture/main.html#lifecyle-of-records">Lifecyle of records</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../design/main.html">Detail Design</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../design/main.html#consistency">Consistency</a></li>
<li class="toctree-l2"><a class="reference internal" href="../design/main.html#streaming-reads">Streaming Reads</a></li>
<li class="toctree-l2"><a class="reference internal" href="../design/main.html#logsegment-lifecycle">LogSegment Lifecycle</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../globalreplicatedlog/main.html">Global Replicated Log</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#region-aware-data-placement-policy">Region Aware Data Placement Policy</a></li>
<li class="toctree-l2"><a class="reference internal" href="../globalreplicatedlog/main.html#cross-region-speculative-reads">Cross Region Speculative Reads</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../implementation/main.html">Implementation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../implementation/storage.html">Storage</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../operations/main.html">Deployment &amp; Administration</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../operations/deployment.html">Cluster Setup &amp; Deployment</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/operations.html">DistributedLog Operations</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/performance.html">Performance Tuning</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/hardware.html">Hardware</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/monitoring.html">Monitoring</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/zookeeper.html">ZooKeeper</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/bookkeeper.html">BookKeeper</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../performance/main.html">Performance</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="main.html">References</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="configuration.html">Configuration Settings</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="">Metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="features.html">Features</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/main.html">Tutorials</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#basic">Basic</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#messaging">Messaging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#replicated-state-machines">Replicated State Machines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/main.html#analytics">Analytics</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../developer/main.html">Developer</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../developer/release.html">Release</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
</ul>
<span id="last"></span>
</div>
</div>
<!-- <div id="slidebox"> -->
<!-- <button id="slidebox_close" type="button" class="close">&times;</button> -->
<!-- <p>Rate This Page</p> -->
<!-- <div id="rateYo"></div> -->
<!-- <p>Comment</p>
<input type="text" name="comment"></input>
<button>Submit</button> -->
<!-- </div> -->
</div>
</div>
<footer class="footer">
<div class="container-fluid">
<div class="row">
<div class="col-md-10 col-md-offset-1">
<p class="pull-right">
<a href="#">Back to top</a>
<br/>
<div id="sourcelink">
<a href="git@github.com:twitter/distributedlog.git/tree/master/docs/references/metrics.rst"
rel="nofollow">Source</a>
<a href="../_sources/references/metrics.txt"
rel="nofollow">Raw</a>
<a href="../__docbird-build.log"
rel="nofollow">Build Log</a>
<a href="/report/stats/distributedlog:distributedlog"
rel="nofollow">Stats</a>
</div>
</p>
<p>
Built and hosted by <a href="/">DocBird</a>.
</p>
</div>
</div>
</div>
</footer>
<script type="text/javascript" src="../_static/js/docbird.js"></script>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-30775-8']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
<!-- <script type="text/javascript" src="//s/d41d8cd98f00b204e9800998ecf8427e/en_US-tbnx1s-1988229788/6163/97/1.4.3/_/download/batch/com.atlassian.jira.collector.plugin.jira-issue-collector-plugin:issuecollector/com.atlassian.jira.collector.plugin.jira-issue-collector-plugin:issuecollector.js?collectorId=e62237fc"></script>
-->
<script type="text/javascript">
$(document).ready(function () {
// track user activity time (from https://github.com/jasonzissman/TimeMe.js)
TimeMe.setIdleDurationInSeconds(30);
TimeMe.setCurrentPageName("my-home-page");
TimeMe.initialize();
// record page visit event when user leaves the page
window.onbeforeunload = function (event) {
xmlhttp=new XMLHttpRequest();
xmlhttp.withCredentials = true;
xmlhttp.open("POST", "/event/distributedlog:distributedlog/visit", false);
xmlhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
var event_data = {
total_time_reading: TimeMe.getTimeOnCurrentPageInSeconds(),
page: window.location.href
};
//alert("send: " + $.param(event_data));
xmlhttp.send($.param(event_data));
};
// ask user for page rating after 20 seconds
// setTimeout(function(){
// alert("Rate this page!");
// }, 20000);
});
</script>
<!-- <style>
#slidebox{
width: 250px;
height: 90px;
padding: 10px;
background-color: #fff;
border: 1px solid #ccc;
position: fixed;
bottom: 3px;
right: -280px;
z-index: 1;
}
#slidebox .close{
margin-top: -5px;
opacity: 0.5;
}
#slidebox .close:hover{
opacity: 0.7;
}
</style> -->
<script type="text/javascript">
$(function() {
// $(window).scroll(function(){
// var distanceTop = $('#last').offset().top - $(window).height();
// if ($(window).scrollTop() > distanceTop)
// $('#slidebox').animate({'right':'3px'},300);
// else
// $('#slidebox').stop(true).animate({'right':'-280px'},100);
// });
// $('#slidebox .close').bind('click',function(){
// $(this).parent().remove();
// });
$("#rateYo").rateYo({
normalFill: "#A0A0A0",
halfStar: true,
rating: (Cookies.get('docbird.rating.distributedlog.distributedlog') || 0.0)
}).on("rateyo.set", function (e, data) {
var event_data = {
comment: '', // see todo note below
rating: data.rating,
page: window.location.href
};
Cookies.get('docbird.rating.distributedlog.distributedlog', data.rating)
$.post('/event/distributedlog:distributedlog/rating', event_data)
// xmlhttp=new XMLHttpRequest();
// xmlhttp.withCredentials = true;
// var event_data = {
// comment: '', // see todo note below
// rating: data.rating,
// page: window.location.href
// };
// xmlhttp.open("GET", "/event/distributedlog/rating?" + $.param(event_data), false);
// xmlhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
// // todo: implement comment form in rating slide out,
// // and instead of hooking this event, include a submit button,
// // and read the rating with rating() method
// // alert("send: " + $.param(event_data));
// xmlhttp.send();
});
});
</script>
<script src="_static/js/selection-sharer.js"></script>
<script>
$('.db-content-body').selectionSharer();
</script>
</body>
</html>