Google Git
Sign in
apache / hbase-site / e5d2e6a6f9646196617268912ffe3d020e593a18 / . / apidocs / src-html / org / apache / hadoop / hbase / regionserver / RegionServerAccounting.html
blob: 6a32349aac87235fcb81158204fb23d2b98ffe6a [file] [log] [blame]
<!DOCTYPE HTML>
<html lang="en">
<head>
<!-- Generated by javadoc (17) -->
<title>Source code</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="source: package: org.apache.hadoop.hbase.regionserver, class: RegionServerAccounting">
<meta name="generator" content="javadoc/SourceToHTMLConverter">
<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style">
</head>
<body class="source-page">
<main role="main">
<div class="source-container">
<pre><span class="source-line-no">001</span><span id="line-1">/*</span>
<span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span>
<span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span>
<span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span>
<span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span>
<span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span>
<span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span>
<span class="source-line-no">009</span><span id="line-9"> *</span>
<span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="source-line-no">011</span><span id="line-11"> *</span>
<span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span>
<span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span>
<span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span>
<span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span>
<span class="source-line-no">017</span><span id="line-17"> */</span>
<span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.regionserver;</span>
<span class="source-line-no">019</span><span id="line-19"></span>
<span class="source-line-no">020</span><span id="line-20">import java.lang.management.MemoryType;</span>
<span class="source-line-no">021</span><span id="line-21">import java.util.concurrent.ConcurrentHashMap;</span>
<span class="source-line-no">022</span><span id="line-22">import java.util.concurrent.ConcurrentMap;</span>
<span class="source-line-no">023</span><span id="line-23">import java.util.concurrent.atomic.LongAdder;</span>
<span class="source-line-no">024</span><span id="line-24">import org.apache.hadoop.conf.Configuration;</span>
<span class="source-line-no">025</span><span id="line-25">import org.apache.hadoop.hbase.io.util.MemorySizeUtil;</span>
<span class="source-line-no">026</span><span id="line-26">import org.apache.hadoop.hbase.util.Pair;</span>
<span class="source-line-no">027</span><span id="line-27">import org.apache.yetus.audience.InterfaceAudience;</span>
<span class="source-line-no">028</span><span id="line-28"></span>
<span class="source-line-no">029</span><span id="line-29">/**</span>
<span class="source-line-no">030</span><span id="line-30"> * RegionServerAccounting keeps record of some basic real time information about the Region Server.</span>
<span class="source-line-no">031</span><span id="line-31"> * Currently, it keeps record the global memstore size and global memstore on-heap and off-heap</span>
<span class="source-line-no">032</span><span id="line-32"> * overhead. It also tracks the replay edits per region.</span>
<span class="source-line-no">033</span><span id="line-33"> */</span>
<span class="source-line-no">034</span><span id="line-34">@InterfaceAudience.Private</span>
<span class="source-line-no">035</span><span id="line-35">public class RegionServerAccounting {</span>
<span class="source-line-no">036</span><span id="line-36"> // memstore data size</span>
<span class="source-line-no">037</span><span id="line-37"> private final LongAdder globalMemStoreDataSize = new LongAdder();</span>
<span class="source-line-no">038</span><span id="line-38"> // memstore heap size.</span>
<span class="source-line-no">039</span><span id="line-39"> private final LongAdder globalMemStoreHeapSize = new LongAdder();</span>
<span class="source-line-no">040</span><span id="line-40"> // memstore off-heap size.</span>
<span class="source-line-no">041</span><span id="line-41"> private final LongAdder globalMemStoreOffHeapSize = new LongAdder();</span>
<span class="source-line-no">042</span><span id="line-42"></span>
<span class="source-line-no">043</span><span id="line-43"> private long globalMemStoreLimit;</span>
<span class="source-line-no">044</span><span id="line-44"> private final float globalMemStoreLimitLowMarkPercent;</span>
<span class="source-line-no">045</span><span id="line-45"> private long globalMemStoreLimitLowMark;</span>
<span class="source-line-no">046</span><span id="line-46"> private final MemoryType memType;</span>
<span class="source-line-no">047</span><span id="line-47"> private long globalOnHeapMemstoreLimit;</span>
<span class="source-line-no">048</span><span id="line-48"> private long globalOnHeapMemstoreLimitLowMark;</span>
<span class="source-line-no">049</span><span id="line-49"></span>
<span class="source-line-no">050</span><span id="line-50"> // encoded region name -&gt; Pair -&gt; read count as first, write count as second.</span>
<span class="source-line-no">051</span><span id="line-51"> // when region close and target rs is the current server, we will put an entry,</span>
<span class="source-line-no">052</span><span id="line-52"> // and will remove it when reigon open after recover them.</span>
<span class="source-line-no">053</span><span id="line-53"> private ConcurrentMap&lt;String, Pair&lt;Long, Long&gt;&gt; retainedRegionRWRequestsCnt;</span>
<span class="source-line-no">054</span><span id="line-54"></span>
<span class="source-line-no">055</span><span id="line-55"> public RegionServerAccounting(Configuration conf) {</span>
<span class="source-line-no">056</span><span id="line-56"> Pair&lt;Long, MemoryType&gt; globalMemstoreSizePair = MemorySizeUtil.getGlobalMemStoreSize(conf);</span>
<span class="source-line-no">057</span><span id="line-57"> this.globalMemStoreLimit = globalMemstoreSizePair.getFirst();</span>
<span class="source-line-no">058</span><span id="line-58"> this.memType = globalMemstoreSizePair.getSecond();</span>
<span class="source-line-no">059</span><span id="line-59"> this.globalMemStoreLimitLowMarkPercent =</span>
<span class="source-line-no">060</span><span id="line-60"> MemorySizeUtil.getGlobalMemStoreHeapLowerMark(conf, this.memType == MemoryType.HEAP);</span>
<span class="source-line-no">061</span><span id="line-61"> // When off heap memstore in use we configure the global off heap space for memstore as bytes</span>
<span class="source-line-no">062</span><span id="line-62"> // not as % of max memory size. In such case, the lower water mark should be specified using the</span>
<span class="source-line-no">063</span><span id="line-63"> // key "hbase.regionserver.global.memstore.size.lower.limit" which says % of the global upper</span>
<span class="source-line-no">064</span><span id="line-64"> // bound and defaults to 95%. In on heap case also specifying this way is ideal. But in the past</span>
<span class="source-line-no">065</span><span id="line-65"> // we used to take lower bound also as the % of xmx (38% as default). For backward compatibility</span>
<span class="source-line-no">066</span><span id="line-66"> // for this deprecated config,we will fall back to read that config when new one is missing.</span>
<span class="source-line-no">067</span><span id="line-67"> // Only for on heap case, do this fallback mechanism. For off heap it makes no sense.</span>
<span class="source-line-no">068</span><span id="line-68"> // TODO When to get rid of the deprecated config? ie</span>
<span class="source-line-no">069</span><span id="line-69"> // "hbase.regionserver.global.memstore.lowerLimit". Can get rid of this boolean passing then.</span>
<span class="source-line-no">070</span><span id="line-70"> this.globalMemStoreLimitLowMark =</span>
<span class="source-line-no">071</span><span id="line-71"> (long) (this.globalMemStoreLimit * this.globalMemStoreLimitLowMarkPercent);</span>
<span class="source-line-no">072</span><span id="line-72"> this.globalOnHeapMemstoreLimit = MemorySizeUtil.getOnheapGlobalMemStoreSize(conf);</span>
<span class="source-line-no">073</span><span id="line-73"> this.globalOnHeapMemstoreLimitLowMark =</span>
<span class="source-line-no">074</span><span id="line-74"> (long) (this.globalOnHeapMemstoreLimit * this.globalMemStoreLimitLowMarkPercent);</span>
<span class="source-line-no">075</span><span id="line-75"> this.retainedRegionRWRequestsCnt = new ConcurrentHashMap&lt;&gt;();</span>
<span class="source-line-no">076</span><span id="line-76"> }</span>
<span class="source-line-no">077</span><span id="line-77"></span>
<span class="source-line-no">078</span><span id="line-78"> long getGlobalMemStoreLimit() {</span>
<span class="source-line-no">079</span><span id="line-79"> return this.globalMemStoreLimit;</span>
<span class="source-line-no">080</span><span id="line-80"> }</span>
<span class="source-line-no">081</span><span id="line-81"></span>
<span class="source-line-no">082</span><span id="line-82"> long getGlobalOffHeapMemStoreLimit() {</span>
<span class="source-line-no">083</span><span id="line-83"> if (isOffheap()) {</span>
<span class="source-line-no">084</span><span id="line-84"> return this.globalMemStoreLimit;</span>
<span class="source-line-no">085</span><span id="line-85"> } else {</span>
<span class="source-line-no">086</span><span id="line-86"> return 0;</span>
<span class="source-line-no">087</span><span id="line-87"> }</span>
<span class="source-line-no">088</span><span id="line-88"> }</span>
<span class="source-line-no">089</span><span id="line-89"></span>
<span class="source-line-no">090</span><span id="line-90"> long getGlobalOnHeapMemStoreLimit() {</span>
<span class="source-line-no">091</span><span id="line-91"> return this.globalOnHeapMemstoreLimit;</span>
<span class="source-line-no">092</span><span id="line-92"> }</span>
<span class="source-line-no">093</span><span id="line-93"></span>
<span class="source-line-no">094</span><span id="line-94"> // Called by the tuners.</span>
<span class="source-line-no">095</span><span id="line-95"> void setGlobalMemStoreLimits(long newGlobalMemstoreLimit) {</span>
<span class="source-line-no">096</span><span id="line-96"> if (this.memType == MemoryType.HEAP) {</span>
<span class="source-line-no">097</span><span id="line-97"> this.globalMemStoreLimit = newGlobalMemstoreLimit;</span>
<span class="source-line-no">098</span><span id="line-98"> this.globalMemStoreLimitLowMark =</span>
<span class="source-line-no">099</span><span id="line-99"> (long) (this.globalMemStoreLimit * this.globalMemStoreLimitLowMarkPercent);</span>
<span class="source-line-no">100</span><span id="line-100"> } else {</span>
<span class="source-line-no">101</span><span id="line-101"> this.globalOnHeapMemstoreLimit = newGlobalMemstoreLimit;</span>
<span class="source-line-no">102</span><span id="line-102"> this.globalOnHeapMemstoreLimitLowMark =</span>
<span class="source-line-no">103</span><span id="line-103"> (long) (this.globalOnHeapMemstoreLimit * this.globalMemStoreLimitLowMarkPercent);</span>
<span class="source-line-no">104</span><span id="line-104"> }</span>
<span class="source-line-no">105</span><span id="line-105"> }</span>
<span class="source-line-no">106</span><span id="line-106"></span>
<span class="source-line-no">107</span><span id="line-107"> boolean isOffheap() {</span>
<span class="source-line-no">108</span><span id="line-108"> return this.memType == MemoryType.NON_HEAP;</span>
<span class="source-line-no">109</span><span id="line-109"> }</span>
<span class="source-line-no">110</span><span id="line-110"></span>
<span class="source-line-no">111</span><span id="line-111"> long getGlobalMemStoreLimitLowMark() {</span>
<span class="source-line-no">112</span><span id="line-112"> return this.globalMemStoreLimitLowMark;</span>
<span class="source-line-no">113</span><span id="line-113"> }</span>
<span class="source-line-no">114</span><span id="line-114"></span>
<span class="source-line-no">115</span><span id="line-115"> float getGlobalMemStoreLimitLowMarkPercent() {</span>
<span class="source-line-no">116</span><span id="line-116"> return this.globalMemStoreLimitLowMarkPercent;</span>
<span class="source-line-no">117</span><span id="line-117"> }</span>
<span class="source-line-no">118</span><span id="line-118"></span>
<span class="source-line-no">119</span><span id="line-119"> /** Returns the global Memstore data size in the RegionServer */</span>
<span class="source-line-no">120</span><span id="line-120"> public long getGlobalMemStoreDataSize() {</span>
<span class="source-line-no">121</span><span id="line-121"> return globalMemStoreDataSize.sum();</span>
<span class="source-line-no">122</span><span id="line-122"> }</span>
<span class="source-line-no">123</span><span id="line-123"></span>
<span class="source-line-no">124</span><span id="line-124"> /** Returns the global memstore on-heap size in the RegionServer */</span>
<span class="source-line-no">125</span><span id="line-125"> public long getGlobalMemStoreHeapSize() {</span>
<span class="source-line-no">126</span><span id="line-126"> return this.globalMemStoreHeapSize.sum();</span>
<span class="source-line-no">127</span><span id="line-127"> }</span>
<span class="source-line-no">128</span><span id="line-128"></span>
<span class="source-line-no">129</span><span id="line-129"> /** Returns the global memstore off-heap size in the RegionServer */</span>
<span class="source-line-no">130</span><span id="line-130"> public long getGlobalMemStoreOffHeapSize() {</span>
<span class="source-line-no">131</span><span id="line-131"> return this.globalMemStoreOffHeapSize.sum();</span>
<span class="source-line-no">132</span><span id="line-132"> }</span>
<span class="source-line-no">133</span><span id="line-133"></span>
<span class="source-line-no">134</span><span id="line-134"> /** Returns the retained metrics of region's read and write requests count */</span>
<span class="source-line-no">135</span><span id="line-135"> protected ConcurrentMap&lt;String, Pair&lt;Long, Long&gt;&gt; getRetainedRegionRWRequestsCnt() {</span>
<span class="source-line-no">136</span><span id="line-136"> return this.retainedRegionRWRequestsCnt;</span>
<span class="source-line-no">137</span><span id="line-137"> }</span>
<span class="source-line-no">138</span><span id="line-138"></span>
<span class="source-line-no">139</span><span id="line-139"> void incGlobalMemStoreSize(MemStoreSize mss) {</span>
<span class="source-line-no">140</span><span id="line-140"> incGlobalMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());</span>
<span class="source-line-no">141</span><span id="line-141"> }</span>
<span class="source-line-no">142</span><span id="line-142"></span>
<span class="source-line-no">143</span><span id="line-143"> public void incGlobalMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {</span>
<span class="source-line-no">144</span><span id="line-144"> globalMemStoreDataSize.add(dataSizeDelta);</span>
<span class="source-line-no">145</span><span id="line-145"> globalMemStoreHeapSize.add(heapSizeDelta);</span>
<span class="source-line-no">146</span><span id="line-146"> globalMemStoreOffHeapSize.add(offHeapSizeDelta);</span>
<span class="source-line-no">147</span><span id="line-147"> }</span>
<span class="source-line-no">148</span><span id="line-148"></span>
<span class="source-line-no">149</span><span id="line-149"> public void decGlobalMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {</span>
<span class="source-line-no">150</span><span id="line-150"> globalMemStoreDataSize.add(-dataSizeDelta);</span>
<span class="source-line-no">151</span><span id="line-151"> globalMemStoreHeapSize.add(-heapSizeDelta);</span>
<span class="source-line-no">152</span><span id="line-152"> globalMemStoreOffHeapSize.add(-offHeapSizeDelta);</span>
<span class="source-line-no">153</span><span id="line-153"> }</span>
<span class="source-line-no">154</span><span id="line-154"></span>
<span class="source-line-no">155</span><span id="line-155"> /**</span>
<span class="source-line-no">156</span><span id="line-156"> * Return the FlushType if we are above the memstore high water mark</span>
<span class="source-line-no">157</span><span id="line-157"> * @return the FlushType</span>
<span class="source-line-no">158</span><span id="line-158"> */</span>
<span class="source-line-no">159</span><span id="line-159"> public FlushType isAboveHighWaterMark() {</span>
<span class="source-line-no">160</span><span id="line-160"> // for onheap memstore we check if the global memstore size and the</span>
<span class="source-line-no">161</span><span id="line-161"> // global heap overhead is greater than the global memstore limit</span>
<span class="source-line-no">162</span><span id="line-162"> if (memType == MemoryType.HEAP) {</span>
<span class="source-line-no">163</span><span id="line-163"> if (getGlobalMemStoreHeapSize() &gt;= globalMemStoreLimit) {</span>
<span class="source-line-no">164</span><span id="line-164"> return FlushType.ABOVE_ONHEAP_HIGHER_MARK;</span>
<span class="source-line-no">165</span><span id="line-165"> }</span>
<span class="source-line-no">166</span><span id="line-166"> } else {</span>
<span class="source-line-no">167</span><span id="line-167"> // If the configured memstore is offheap, check for two things</span>
<span class="source-line-no">168</span><span id="line-168"> // 1) If the global memstore off-heap size is greater than the configured</span>
<span class="source-line-no">169</span><span id="line-169"> // 'hbase.regionserver.offheap.global.memstore.size'</span>
<span class="source-line-no">170</span><span id="line-170"> // 2) If the global memstore heap size is greater than the configured onheap</span>
<span class="source-line-no">171</span><span id="line-171"> // global memstore limit 'hbase.regionserver.global.memstore.size'.</span>
<span class="source-line-no">172</span><span id="line-172"> // We do this to avoid OOME incase of scenarios where the heap is occupied with</span>
<span class="source-line-no">173</span><span id="line-173"> // lot of onheap references to the cells in memstore</span>
<span class="source-line-no">174</span><span id="line-174"> if (getGlobalMemStoreOffHeapSize() &gt;= globalMemStoreLimit) {</span>
<span class="source-line-no">175</span><span id="line-175"> // Indicates that global memstore size is above the configured</span>
<span class="source-line-no">176</span><span id="line-176"> // 'hbase.regionserver.offheap.global.memstore.size'</span>
<span class="source-line-no">177</span><span id="line-177"> return FlushType.ABOVE_OFFHEAP_HIGHER_MARK;</span>
<span class="source-line-no">178</span><span id="line-178"> } else if (getGlobalMemStoreHeapSize() &gt;= this.globalOnHeapMemstoreLimit) {</span>
<span class="source-line-no">179</span><span id="line-179"> // Indicates that the offheap memstore's heap overhead is greater than the</span>
<span class="source-line-no">180</span><span id="line-180"> // configured 'hbase.regionserver.global.memstore.size'.</span>
<span class="source-line-no">181</span><span id="line-181"> return FlushType.ABOVE_ONHEAP_HIGHER_MARK;</span>
<span class="source-line-no">182</span><span id="line-182"> }</span>
<span class="source-line-no">183</span><span id="line-183"> }</span>
<span class="source-line-no">184</span><span id="line-184"> return FlushType.NORMAL;</span>
<span class="source-line-no">185</span><span id="line-185"> }</span>
<span class="source-line-no">186</span><span id="line-186"></span>
<span class="source-line-no">187</span><span id="line-187"> /**</span>
<span class="source-line-no">188</span><span id="line-188"> * Return the FlushType if we're above the low watermark</span>
<span class="source-line-no">189</span><span id="line-189"> * @return the FlushType</span>
<span class="source-line-no">190</span><span id="line-190"> */</span>
<span class="source-line-no">191</span><span id="line-191"> public FlushType isAboveLowWaterMark() {</span>
<span class="source-line-no">192</span><span id="line-192"> // for onheap memstore we check if the global memstore size and the</span>
<span class="source-line-no">193</span><span id="line-193"> // global heap overhead is greater than the global memstore lower mark limit</span>
<span class="source-line-no">194</span><span id="line-194"> if (memType == MemoryType.HEAP) {</span>
<span class="source-line-no">195</span><span id="line-195"> if (getGlobalMemStoreHeapSize() &gt;= globalMemStoreLimitLowMark) {</span>
<span class="source-line-no">196</span><span id="line-196"> return FlushType.ABOVE_ONHEAP_LOWER_MARK;</span>
<span class="source-line-no">197</span><span id="line-197"> }</span>
<span class="source-line-no">198</span><span id="line-198"> } else {</span>
<span class="source-line-no">199</span><span id="line-199"> if (getGlobalMemStoreOffHeapSize() &gt;= globalMemStoreLimitLowMark) {</span>
<span class="source-line-no">200</span><span id="line-200"> // Indicates that the offheap memstore's size is greater than the global memstore</span>
<span class="source-line-no">201</span><span id="line-201"> // lower limit</span>
<span class="source-line-no">202</span><span id="line-202"> return FlushType.ABOVE_OFFHEAP_LOWER_MARK;</span>
<span class="source-line-no">203</span><span id="line-203"> } else if (getGlobalMemStoreHeapSize() &gt;= globalOnHeapMemstoreLimitLowMark) {</span>
<span class="source-line-no">204</span><span id="line-204"> // Indicates that the offheap memstore's heap overhead is greater than the global memstore</span>
<span class="source-line-no">205</span><span id="line-205"> // onheap lower limit</span>
<span class="source-line-no">206</span><span id="line-206"> return FlushType.ABOVE_ONHEAP_LOWER_MARK;</span>
<span class="source-line-no">207</span><span id="line-207"> }</span>
<span class="source-line-no">208</span><span id="line-208"> }</span>
<span class="source-line-no">209</span><span id="line-209"> return FlushType.NORMAL;</span>
<span class="source-line-no">210</span><span id="line-210"> }</span>
<span class="source-line-no">211</span><span id="line-211"></span>
<span class="source-line-no">212</span><span id="line-212"> /**</span>
<span class="source-line-no">213</span><span id="line-213"> * @return the flush pressure of all stores on this regionserver. The value should be greater than</span>
<span class="source-line-no">214</span><span id="line-214"> * or equal to 0.0, and any value greater than 1.0 means we enter the emergency state that</span>
<span class="source-line-no">215</span><span id="line-215"> * global memstore size already exceeds lower limit.</span>
<span class="source-line-no">216</span><span id="line-216"> */</span>
<span class="source-line-no">217</span><span id="line-217"> public double getFlushPressure() {</span>
<span class="source-line-no">218</span><span id="line-218"> if (memType == MemoryType.HEAP) {</span>
<span class="source-line-no">219</span><span id="line-219"> return (getGlobalMemStoreHeapSize()) * 1.0 / globalMemStoreLimitLowMark;</span>
<span class="source-line-no">220</span><span id="line-220"> } else {</span>
<span class="source-line-no">221</span><span id="line-221"> return Math.max(getGlobalMemStoreOffHeapSize() * 1.0 / globalMemStoreLimitLowMark,</span>
<span class="source-line-no">222</span><span id="line-222"> getGlobalMemStoreHeapSize() * 1.0 / globalOnHeapMemstoreLimitLowMark);</span>
<span class="source-line-no">223</span><span id="line-223"> }</span>
<span class="source-line-no">224</span><span id="line-224"> }</span>
<span class="source-line-no">225</span><span id="line-225">}</span>
</pre>
</div>
</main>
</body>
</html>