| <!DOCTYPE HTML> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (17) --> |
| <title>Source code</title> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="source: package: org.apache.hadoop.hbase.tool, class: CanaryTool, class: RegionServerStdOutSink"> |
| <meta name="generator" content="javadoc/SourceToHTMLConverter"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| </head> |
| <body class="source-page"> |
| <main role="main"> |
| <div class="source-container"> |
| <pre><span class="source-line-no">001</span><span id="line-1">/*</span> |
| <span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span> |
| <span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span> |
| <span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span> |
| <span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span> |
| <span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span> |
| <span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span> |
| <span class="source-line-no">009</span><span id="line-9"> *</span> |
| <span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="source-line-no">011</span><span id="line-11"> *</span> |
| <span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span> |
| <span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span> |
| <span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> |
| <span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span> |
| <span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span> |
| <span class="source-line-no">017</span><span id="line-17"> */</span> |
| <span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.tool;</span> |
| <span class="source-line-no">019</span><span id="line-19"></span> |
| <span class="source-line-no">020</span><span id="line-20">import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT;</span> |
| <span class="source-line-no">021</span><span id="line-21">import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;</span> |
| <span class="source-line-no">022</span><span id="line-22">import static org.apache.hadoop.hbase.util.Addressing.inetSocketAddress2String;</span> |
| <span class="source-line-no">023</span><span id="line-23"></span> |
| <span class="source-line-no">024</span><span id="line-24">import java.io.Closeable;</span> |
| <span class="source-line-no">025</span><span id="line-25">import java.io.IOException;</span> |
| <span class="source-line-no">026</span><span id="line-26">import java.net.BindException;</span> |
| <span class="source-line-no">027</span><span id="line-27">import java.net.InetSocketAddress;</span> |
| <span class="source-line-no">028</span><span id="line-28">import java.util.ArrayList;</span> |
| <span class="source-line-no">029</span><span id="line-29">import java.util.Arrays;</span> |
| <span class="source-line-no">030</span><span id="line-30">import java.util.Collections;</span> |
| <span class="source-line-no">031</span><span id="line-31">import java.util.EnumSet;</span> |
| <span class="source-line-no">032</span><span id="line-32">import java.util.HashMap;</span> |
| <span class="source-line-no">033</span><span id="line-33">import java.util.HashSet;</span> |
| <span class="source-line-no">034</span><span id="line-34">import java.util.LinkedList;</span> |
| <span class="source-line-no">035</span><span id="line-35">import java.util.List;</span> |
| <span class="source-line-no">036</span><span id="line-36">import java.util.Map;</span> |
| <span class="source-line-no">037</span><span id="line-37">import java.util.Set;</span> |
| <span class="source-line-no">038</span><span id="line-38">import java.util.TreeSet;</span> |
| <span class="source-line-no">039</span><span id="line-39">import java.util.concurrent.Callable;</span> |
| <span class="source-line-no">040</span><span id="line-40">import java.util.concurrent.ConcurrentHashMap;</span> |
| <span class="source-line-no">041</span><span id="line-41">import java.util.concurrent.ConcurrentMap;</span> |
| <span class="source-line-no">042</span><span id="line-42">import java.util.concurrent.ExecutionException;</span> |
| <span class="source-line-no">043</span><span id="line-43">import java.util.concurrent.ExecutorService;</span> |
| <span class="source-line-no">044</span><span id="line-44">import java.util.concurrent.Future;</span> |
| <span class="source-line-no">045</span><span id="line-45">import java.util.concurrent.ScheduledThreadPoolExecutor;</span> |
| <span class="source-line-no">046</span><span id="line-46">import java.util.concurrent.ThreadLocalRandom;</span> |
| <span class="source-line-no">047</span><span id="line-47">import java.util.concurrent.atomic.AtomicLong;</span> |
| <span class="source-line-no">048</span><span id="line-48">import java.util.concurrent.atomic.LongAdder;</span> |
| <span class="source-line-no">049</span><span id="line-49">import java.util.regex.Matcher;</span> |
| <span class="source-line-no">050</span><span id="line-50">import java.util.regex.Pattern;</span> |
| <span class="source-line-no">051</span><span id="line-51">import org.apache.commons.lang3.time.StopWatch;</span> |
| <span class="source-line-no">052</span><span id="line-52">import org.apache.hadoop.conf.Configuration;</span> |
| <span class="source-line-no">053</span><span id="line-53">import org.apache.hadoop.hbase.AuthUtil;</span> |
| <span class="source-line-no">054</span><span id="line-54">import org.apache.hadoop.hbase.ChoreService;</span> |
| <span class="source-line-no">055</span><span id="line-55">import org.apache.hadoop.hbase.ClusterMetrics;</span> |
| <span class="source-line-no">056</span><span id="line-56">import org.apache.hadoop.hbase.ClusterMetrics.Option;</span> |
| <span class="source-line-no">057</span><span id="line-57">import org.apache.hadoop.hbase.DoNotRetryIOException;</span> |
| <span class="source-line-no">058</span><span id="line-58">import org.apache.hadoop.hbase.HBaseConfiguration;</span> |
| <span class="source-line-no">059</span><span id="line-59">import org.apache.hadoop.hbase.HBaseInterfaceAudience;</span> |
| <span class="source-line-no">060</span><span id="line-60">import org.apache.hadoop.hbase.HConstants;</span> |
| <span class="source-line-no">061</span><span id="line-61">import org.apache.hadoop.hbase.HRegionLocation;</span> |
| <span class="source-line-no">062</span><span id="line-62">import org.apache.hadoop.hbase.MetaTableAccessor;</span> |
| <span class="source-line-no">063</span><span id="line-63">import org.apache.hadoop.hbase.NamespaceDescriptor;</span> |
| <span class="source-line-no">064</span><span id="line-64">import org.apache.hadoop.hbase.ScheduledChore;</span> |
| <span class="source-line-no">065</span><span id="line-65">import org.apache.hadoop.hbase.ServerName;</span> |
| <span class="source-line-no">066</span><span id="line-66">import org.apache.hadoop.hbase.TableName;</span> |
| <span class="source-line-no">067</span><span id="line-67">import org.apache.hadoop.hbase.TableNotEnabledException;</span> |
| <span class="source-line-no">068</span><span id="line-68">import org.apache.hadoop.hbase.TableNotFoundException;</span> |
| <span class="source-line-no">069</span><span id="line-69">import org.apache.hadoop.hbase.client.Admin;</span> |
| <span class="source-line-no">070</span><span id="line-70">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span> |
| <span class="source-line-no">071</span><span id="line-71">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span> |
| <span class="source-line-no">072</span><span id="line-72">import org.apache.hadoop.hbase.client.Connection;</span> |
| <span class="source-line-no">073</span><span id="line-73">import org.apache.hadoop.hbase.client.ConnectionFactory;</span> |
| <span class="source-line-no">074</span><span id="line-74">import org.apache.hadoop.hbase.client.Get;</span> |
| <span class="source-line-no">075</span><span id="line-75">import org.apache.hadoop.hbase.client.Put;</span> |
| <span class="source-line-no">076</span><span id="line-76">import org.apache.hadoop.hbase.client.RegionInfo;</span> |
| <span class="source-line-no">077</span><span id="line-77">import org.apache.hadoop.hbase.client.RegionLocator;</span> |
| <span class="source-line-no">078</span><span id="line-78">import org.apache.hadoop.hbase.client.ResultScanner;</span> |
| <span class="source-line-no">079</span><span id="line-79">import org.apache.hadoop.hbase.client.Scan;</span> |
| <span class="source-line-no">080</span><span id="line-80">import org.apache.hadoop.hbase.client.Table;</span> |
| <span class="source-line-no">081</span><span id="line-81">import org.apache.hadoop.hbase.client.TableDescriptor;</span> |
| <span class="source-line-no">082</span><span id="line-82">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span> |
| <span class="source-line-no">083</span><span id="line-83">import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;</span> |
| <span class="source-line-no">084</span><span id="line-84">import org.apache.hadoop.hbase.http.InfoServer;</span> |
| <span class="source-line-no">085</span><span id="line-85">import org.apache.hadoop.hbase.tool.CanaryTool.RegionTask.TaskType;</span> |
| <span class="source-line-no">086</span><span id="line-86">import org.apache.hadoop.hbase.util.Bytes;</span> |
| <span class="source-line-no">087</span><span id="line-87">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span> |
| <span class="source-line-no">088</span><span id="line-88">import org.apache.hadoop.hbase.util.Pair;</span> |
| <span class="source-line-no">089</span><span id="line-89">import org.apache.hadoop.hbase.util.ReflectionUtils;</span> |
| <span class="source-line-no">090</span><span id="line-90">import org.apache.hadoop.hbase.util.RegionSplitter;</span> |
| <span class="source-line-no">091</span><span id="line-91">import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;</span> |
| <span class="source-line-no">092</span><span id="line-92">import org.apache.hadoop.hbase.zookeeper.ZKConfig;</span> |
| <span class="source-line-no">093</span><span id="line-93">import org.apache.hadoop.util.Tool;</span> |
| <span class="source-line-no">094</span><span id="line-94">import org.apache.hadoop.util.ToolRunner;</span> |
| <span class="source-line-no">095</span><span id="line-95">import org.apache.yetus.audience.InterfaceAudience;</span> |
| <span class="source-line-no">096</span><span id="line-96">import org.apache.zookeeper.KeeperException;</span> |
| <span class="source-line-no">097</span><span id="line-97">import org.apache.zookeeper.ZooKeeper;</span> |
| <span class="source-line-no">098</span><span id="line-98">import org.apache.zookeeper.client.ConnectStringParser;</span> |
| <span class="source-line-no">099</span><span id="line-99">import org.apache.zookeeper.data.Stat;</span> |
| <span class="source-line-no">100</span><span id="line-100">import org.slf4j.Logger;</span> |
| <span class="source-line-no">101</span><span id="line-101">import org.slf4j.LoggerFactory;</span> |
| <span class="source-line-no">102</span><span id="line-102"></span> |
| <span class="source-line-no">103</span><span id="line-103">import org.apache.hbase.thirdparty.com.google.common.collect.Lists;</span> |
| <span class="source-line-no">104</span><span id="line-104"></span> |
| <span class="source-line-no">105</span><span id="line-105">/**</span> |
| <span class="source-line-no">106</span><span id="line-106"> * HBase Canary Tool for "canary monitoring" of a running HBase cluster. There are three modes:</span> |
| <span class="source-line-no">107</span><span id="line-107"> * <ol></span> |
| <span class="source-line-no">108</span><span id="line-108"> * <li>region mode (Default): For each region, try to get one row per column family outputting</span> |
| <span class="source-line-no">109</span><span id="line-109"> * information on failure (ERROR) or else the latency.</li></span> |
| <span class="source-line-no">110</span><span id="line-110"> * <li>regionserver mode: For each regionserver try to get one row from one table selected randomly</span> |
| <span class="source-line-no">111</span><span id="line-111"> * outputting information on failure (ERROR) or else the latency.</li></span> |
| <span class="source-line-no">112</span><span id="line-112"> * <li>zookeeper mode: for each zookeeper instance, selects a znode outputting information on</span> |
| <span class="source-line-no">113</span><span id="line-113"> * failure (ERROR) or else the latency.</li></span> |
| <span class="source-line-no">114</span><span id="line-114"> * </ol></span> |
| <span class="source-line-no">115</span><span id="line-115"> */</span> |
| <span class="source-line-no">116</span><span id="line-116">@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)</span> |
| <span class="source-line-no">117</span><span id="line-117">public class CanaryTool implements Tool, Canary {</span> |
| <span class="source-line-no">118</span><span id="line-118"> public static final String HBASE_CANARY_INFO_PORT = "hbase.canary.info.port";</span> |
| <span class="source-line-no">119</span><span id="line-119"> public static final String HBASE_CANARY_INFO_BINDADDRESS = "hbase.canary.info.bindAddress";</span> |
| <span class="source-line-no">120</span><span id="line-120"></span> |
| <span class="source-line-no">121</span><span id="line-121"> private void putUpWebUI() throws IOException {</span> |
| <span class="source-line-no">122</span><span id="line-122"> int port = conf.getInt(HBASE_CANARY_INFO_PORT, -1);</span> |
| <span class="source-line-no">123</span><span id="line-123"> // -1 is for disabling info server</span> |
| <span class="source-line-no">124</span><span id="line-124"> if (port < 0) {</span> |
| <span class="source-line-no">125</span><span id="line-125"> return;</span> |
| <span class="source-line-no">126</span><span id="line-126"> }</span> |
| <span class="source-line-no">127</span><span id="line-127"> if (zookeeperMode) {</span> |
| <span class="source-line-no">128</span><span id="line-128"> LOG.info("WebUI is not supported in Zookeeper mode");</span> |
| <span class="source-line-no">129</span><span id="line-129"> } else if (regionServerMode) {</span> |
| <span class="source-line-no">130</span><span id="line-130"> LOG.info("WebUI is not supported in RegionServer mode");</span> |
| <span class="source-line-no">131</span><span id="line-131"> } else {</span> |
| <span class="source-line-no">132</span><span id="line-132"> String addr = conf.get(HBASE_CANARY_INFO_BINDADDRESS, "0.0.0.0");</span> |
| <span class="source-line-no">133</span><span id="line-133"> try {</span> |
| <span class="source-line-no">134</span><span id="line-134"> InfoServer infoServer = new InfoServer("canary", addr, port, false, conf);</span> |
| <span class="source-line-no">135</span><span id="line-135"> infoServer.addUnprivilegedServlet("canary", "/canary-status", CanaryStatusServlet.class);</span> |
| <span class="source-line-no">136</span><span id="line-136"> infoServer.setAttribute("sink", getSink(conf, RegionStdOutSink.class));</span> |
| <span class="source-line-no">137</span><span id="line-137"> infoServer.start();</span> |
| <span class="source-line-no">138</span><span id="line-138"> LOG.info("Bind Canary http info server to {}:{} ", addr, port);</span> |
| <span class="source-line-no">139</span><span id="line-139"> } catch (BindException e) {</span> |
| <span class="source-line-no">140</span><span id="line-140"> LOG.warn("Failed binding Canary http info server to {}:{}", addr, port, e);</span> |
| <span class="source-line-no">141</span><span id="line-141"> }</span> |
| <span class="source-line-no">142</span><span id="line-142"> }</span> |
| <span class="source-line-no">143</span><span id="line-143"> }</span> |
| <span class="source-line-no">144</span><span id="line-144"></span> |
| <span class="source-line-no">145</span><span id="line-145"> @Override</span> |
| <span class="source-line-no">146</span><span id="line-146"> public int checkRegions(String[] targets) throws Exception {</span> |
| <span class="source-line-no">147</span><span id="line-147"> String configuredReadTableTimeoutsStr = conf.get(HBASE_CANARY_REGION_READ_TABLE_TIMEOUT);</span> |
| <span class="source-line-no">148</span><span id="line-148"> try {</span> |
| <span class="source-line-no">149</span><span id="line-149"> LOG.info("Canary tool is running in Region mode");</span> |
| <span class="source-line-no">150</span><span id="line-150"> if (configuredReadTableTimeoutsStr != null) {</span> |
| <span class="source-line-no">151</span><span id="line-151"> populateReadTableTimeoutsMap(configuredReadTableTimeoutsStr);</span> |
| <span class="source-line-no">152</span><span id="line-152"> }</span> |
| <span class="source-line-no">153</span><span id="line-153"> } catch (IllegalArgumentException e) {</span> |
| <span class="source-line-no">154</span><span id="line-154"> LOG.error("Constructing read table timeouts map failed ", e);</span> |
| <span class="source-line-no">155</span><span id="line-155"> return USAGE_EXIT_CODE;</span> |
| <span class="source-line-no">156</span><span id="line-156"> }</span> |
| <span class="source-line-no">157</span><span id="line-157"> return runMonitor(targets);</span> |
| <span class="source-line-no">158</span><span id="line-158"> }</span> |
| <span class="source-line-no">159</span><span id="line-159"></span> |
| <span class="source-line-no">160</span><span id="line-160"> @Override</span> |
| <span class="source-line-no">161</span><span id="line-161"> public int checkRegionServers(String[] targets) throws Exception {</span> |
| <span class="source-line-no">162</span><span id="line-162"> regionServerMode = true;</span> |
| <span class="source-line-no">163</span><span id="line-163"> LOG.info("Canary tool is running in RegionServer mode");</span> |
| <span class="source-line-no">164</span><span id="line-164"> return runMonitor(targets);</span> |
| <span class="source-line-no">165</span><span id="line-165"> }</span> |
| <span class="source-line-no">166</span><span id="line-166"></span> |
| <span class="source-line-no">167</span><span id="line-167"> @Override</span> |
| <span class="source-line-no">168</span><span id="line-168"> public int checkZooKeeper() throws Exception {</span> |
| <span class="source-line-no">169</span><span id="line-169"> zookeeperMode = true;</span> |
| <span class="source-line-no">170</span><span id="line-170"> LOG.info("Canary tool is running in ZooKeeper mode");</span> |
| <span class="source-line-no">171</span><span id="line-171"> return runMonitor(null);</span> |
| <span class="source-line-no">172</span><span id="line-172"> }</span> |
| <span class="source-line-no">173</span><span id="line-173"></span> |
| <span class="source-line-no">174</span><span id="line-174"> /**</span> |
| <span class="source-line-no">175</span><span id="line-175"> * Sink interface used by the canary to output information</span> |
| <span class="source-line-no">176</span><span id="line-176"> */</span> |
| <span class="source-line-no">177</span><span id="line-177"> public interface Sink {</span> |
| <span class="source-line-no">178</span><span id="line-178"> long getReadFailureCount();</span> |
| <span class="source-line-no">179</span><span id="line-179"></span> |
| <span class="source-line-no">180</span><span id="line-180"> long incReadFailureCount();</span> |
| <span class="source-line-no">181</span><span id="line-181"></span> |
| <span class="source-line-no">182</span><span id="line-182"> Map<String, String> getReadFailures();</span> |
| <span class="source-line-no">183</span><span id="line-183"></span> |
| <span class="source-line-no">184</span><span id="line-184"> void updateReadFailures(String regionName, String serverName);</span> |
| <span class="source-line-no">185</span><span id="line-185"></span> |
| <span class="source-line-no">186</span><span id="line-186"> long getWriteFailureCount();</span> |
| <span class="source-line-no">187</span><span id="line-187"></span> |
| <span class="source-line-no">188</span><span id="line-188"> long incWriteFailureCount();</span> |
| <span class="source-line-no">189</span><span id="line-189"></span> |
| <span class="source-line-no">190</span><span id="line-190"> Map<String, String> getWriteFailures();</span> |
| <span class="source-line-no">191</span><span id="line-191"></span> |
| <span class="source-line-no">192</span><span id="line-192"> void updateWriteFailures(String regionName, String serverName);</span> |
| <span class="source-line-no">193</span><span id="line-193"></span> |
| <span class="source-line-no">194</span><span id="line-194"> long getReadSuccessCount();</span> |
| <span class="source-line-no">195</span><span id="line-195"></span> |
| <span class="source-line-no">196</span><span id="line-196"> long incReadSuccessCount();</span> |
| <span class="source-line-no">197</span><span id="line-197"></span> |
| <span class="source-line-no">198</span><span id="line-198"> long getWriteSuccessCount();</span> |
| <span class="source-line-no">199</span><span id="line-199"></span> |
| <span class="source-line-no">200</span><span id="line-200"> long incWriteSuccessCount();</span> |
| <span class="source-line-no">201</span><span id="line-201"></span> |
| <span class="source-line-no">202</span><span id="line-202"> void stop();</span> |
| <span class="source-line-no">203</span><span id="line-203"></span> |
| <span class="source-line-no">204</span><span id="line-204"> boolean isStopped();</span> |
| <span class="source-line-no">205</span><span id="line-205"> }</span> |
| <span class="source-line-no">206</span><span id="line-206"></span> |
| <span class="source-line-no">207</span><span id="line-207"> /**</span> |
| <span class="source-line-no">208</span><span id="line-208"> * Simple implementation of canary sink that allows plotting to a file or standard output.</span> |
| <span class="source-line-no">209</span><span id="line-209"> */</span> |
| <span class="source-line-no">210</span><span id="line-210"> public static class StdOutSink implements Sink {</span> |
| <span class="source-line-no">211</span><span id="line-211"> private AtomicLong readFailureCount = new AtomicLong(0), writeFailureCount = new AtomicLong(0),</span> |
| <span class="source-line-no">212</span><span id="line-212"> readSuccessCount = new AtomicLong(0), writeSuccessCount = new AtomicLong(0);</span> |
| <span class="source-line-no">213</span><span id="line-213"> private Map<String, String> readFailures = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">214</span><span id="line-214"> private Map<String, String> writeFailures = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">215</span><span id="line-215"> private volatile boolean stopped = false;</span> |
| <span class="source-line-no">216</span><span id="line-216"></span> |
| <span class="source-line-no">217</span><span id="line-217"> @Override</span> |
| <span class="source-line-no">218</span><span id="line-218"> public long getReadFailureCount() {</span> |
| <span class="source-line-no">219</span><span id="line-219"> return readFailureCount.get();</span> |
| <span class="source-line-no">220</span><span id="line-220"> }</span> |
| <span class="source-line-no">221</span><span id="line-221"></span> |
| <span class="source-line-no">222</span><span id="line-222"> @Override</span> |
| <span class="source-line-no">223</span><span id="line-223"> public long incReadFailureCount() {</span> |
| <span class="source-line-no">224</span><span id="line-224"> return readFailureCount.incrementAndGet();</span> |
| <span class="source-line-no">225</span><span id="line-225"> }</span> |
| <span class="source-line-no">226</span><span id="line-226"></span> |
| <span class="source-line-no">227</span><span id="line-227"> @Override</span> |
| <span class="source-line-no">228</span><span id="line-228"> public Map<String, String> getReadFailures() {</span> |
| <span class="source-line-no">229</span><span id="line-229"> return readFailures;</span> |
| <span class="source-line-no">230</span><span id="line-230"> }</span> |
| <span class="source-line-no">231</span><span id="line-231"></span> |
| <span class="source-line-no">232</span><span id="line-232"> @Override</span> |
| <span class="source-line-no">233</span><span id="line-233"> public void updateReadFailures(String regionName, String serverName) {</span> |
| <span class="source-line-no">234</span><span id="line-234"> readFailures.put(regionName, serverName);</span> |
| <span class="source-line-no">235</span><span id="line-235"> }</span> |
| <span class="source-line-no">236</span><span id="line-236"></span> |
| <span class="source-line-no">237</span><span id="line-237"> @Override</span> |
| <span class="source-line-no">238</span><span id="line-238"> public long getWriteFailureCount() {</span> |
| <span class="source-line-no">239</span><span id="line-239"> return writeFailureCount.get();</span> |
| <span class="source-line-no">240</span><span id="line-240"> }</span> |
| <span class="source-line-no">241</span><span id="line-241"></span> |
| <span class="source-line-no">242</span><span id="line-242"> @Override</span> |
| <span class="source-line-no">243</span><span id="line-243"> public long incWriteFailureCount() {</span> |
| <span class="source-line-no">244</span><span id="line-244"> return writeFailureCount.incrementAndGet();</span> |
| <span class="source-line-no">245</span><span id="line-245"> }</span> |
| <span class="source-line-no">246</span><span id="line-246"></span> |
| <span class="source-line-no">247</span><span id="line-247"> @Override</span> |
| <span class="source-line-no">248</span><span id="line-248"> public Map<String, String> getWriteFailures() {</span> |
| <span class="source-line-no">249</span><span id="line-249"> return writeFailures;</span> |
| <span class="source-line-no">250</span><span id="line-250"> }</span> |
| <span class="source-line-no">251</span><span id="line-251"></span> |
| <span class="source-line-no">252</span><span id="line-252"> @Override</span> |
| <span class="source-line-no">253</span><span id="line-253"> public void updateWriteFailures(String regionName, String serverName) {</span> |
| <span class="source-line-no">254</span><span id="line-254"> writeFailures.put(regionName, serverName);</span> |
| <span class="source-line-no">255</span><span id="line-255"> }</span> |
| <span class="source-line-no">256</span><span id="line-256"></span> |
| <span class="source-line-no">257</span><span id="line-257"> @Override</span> |
| <span class="source-line-no">258</span><span id="line-258"> public long getReadSuccessCount() {</span> |
| <span class="source-line-no">259</span><span id="line-259"> return readSuccessCount.get();</span> |
| <span class="source-line-no">260</span><span id="line-260"> }</span> |
| <span class="source-line-no">261</span><span id="line-261"></span> |
| <span class="source-line-no">262</span><span id="line-262"> @Override</span> |
| <span class="source-line-no">263</span><span id="line-263"> public long incReadSuccessCount() {</span> |
| <span class="source-line-no">264</span><span id="line-264"> return readSuccessCount.incrementAndGet();</span> |
| <span class="source-line-no">265</span><span id="line-265"> }</span> |
| <span class="source-line-no">266</span><span id="line-266"></span> |
| <span class="source-line-no">267</span><span id="line-267"> @Override</span> |
| <span class="source-line-no">268</span><span id="line-268"> public long getWriteSuccessCount() {</span> |
| <span class="source-line-no">269</span><span id="line-269"> return writeSuccessCount.get();</span> |
| <span class="source-line-no">270</span><span id="line-270"> }</span> |
| <span class="source-line-no">271</span><span id="line-271"></span> |
| <span class="source-line-no">272</span><span id="line-272"> @Override</span> |
| <span class="source-line-no">273</span><span id="line-273"> public long incWriteSuccessCount() {</span> |
| <span class="source-line-no">274</span><span id="line-274"> return writeSuccessCount.incrementAndGet();</span> |
| <span class="source-line-no">275</span><span id="line-275"> }</span> |
| <span class="source-line-no">276</span><span id="line-276"></span> |
| <span class="source-line-no">277</span><span id="line-277"> public void stop() {</span> |
| <span class="source-line-no">278</span><span id="line-278"> stopped = true;</span> |
| <span class="source-line-no">279</span><span id="line-279"> }</span> |
| <span class="source-line-no">280</span><span id="line-280"></span> |
| <span class="source-line-no">281</span><span id="line-281"> @Override</span> |
| <span class="source-line-no">282</span><span id="line-282"> public boolean isStopped() {</span> |
| <span class="source-line-no">283</span><span id="line-283"> return stopped;</span> |
| <span class="source-line-no">284</span><span id="line-284"> }</span> |
| <span class="source-line-no">285</span><span id="line-285"> }</span> |
| <span class="source-line-no">286</span><span id="line-286"></span> |
| <span class="source-line-no">287</span><span id="line-287"> /**</span> |
| <span class="source-line-no">288</span><span id="line-288"> * By RegionServer, for 'regionserver' mode.</span> |
| <span class="source-line-no">289</span><span id="line-289"> */</span> |
| <span class="source-line-no">290</span><span id="line-290"> public static class RegionServerStdOutSink extends StdOutSink {</span> |
| <span class="source-line-no">291</span><span id="line-291"> public void publishReadFailure(String table, String server) {</span> |
| <span class="source-line-no">292</span><span id="line-292"> incReadFailureCount();</span> |
| <span class="source-line-no">293</span><span id="line-293"> LOG.error("Read from {} on {}", table, server);</span> |
| <span class="source-line-no">294</span><span id="line-294"> }</span> |
| <span class="source-line-no">295</span><span id="line-295"></span> |
| <span class="source-line-no">296</span><span id="line-296"> public void publishReadTiming(String table, String server, long msTime) {</span> |
| <span class="source-line-no">297</span><span id="line-297"> LOG.info("Read from {} on {} in {}ms", table, server, msTime);</span> |
| <span class="source-line-no">298</span><span id="line-298"> }</span> |
| <span class="source-line-no">299</span><span id="line-299"> }</span> |
| <span class="source-line-no">300</span><span id="line-300"></span> |
| <span class="source-line-no">301</span><span id="line-301"> /**</span> |
| <span class="source-line-no">302</span><span id="line-302"> * Output for 'zookeeper' mode.</span> |
| <span class="source-line-no">303</span><span id="line-303"> */</span> |
| <span class="source-line-no">304</span><span id="line-304"> public static class ZookeeperStdOutSink extends StdOutSink {</span> |
| <span class="source-line-no">305</span><span id="line-305"> public void publishReadFailure(String znode, String server) {</span> |
| <span class="source-line-no">306</span><span id="line-306"> incReadFailureCount();</span> |
| <span class="source-line-no">307</span><span id="line-307"> LOG.error("Read from {} on {}", znode, server);</span> |
| <span class="source-line-no">308</span><span id="line-308"> }</span> |
| <span class="source-line-no">309</span><span id="line-309"></span> |
| <span class="source-line-no">310</span><span id="line-310"> public void publishReadTiming(String znode, String server, long msTime) {</span> |
| <span class="source-line-no">311</span><span id="line-311"> LOG.info("Read from {} on {} in {}ms", znode, server, msTime);</span> |
| <span class="source-line-no">312</span><span id="line-312"> }</span> |
| <span class="source-line-no">313</span><span id="line-313"> }</span> |
| <span class="source-line-no">314</span><span id="line-314"></span> |
| <span class="source-line-no">315</span><span id="line-315"> /**</span> |
| <span class="source-line-no">316</span><span id="line-316"> * By Region, for 'region' mode.</span> |
| <span class="source-line-no">317</span><span id="line-317"> */</span> |
| <span class="source-line-no">318</span><span id="line-318"> public static class RegionStdOutSink extends StdOutSink {</span> |
| <span class="source-line-no">319</span><span id="line-319"> private Map<String, LongAdder> perTableReadLatency = new HashMap<>();</span> |
| <span class="source-line-no">320</span><span id="line-320"> private LongAdder writeLatency = new LongAdder();</span> |
| <span class="source-line-no">321</span><span id="line-321"> private final ConcurrentMap<String, List<RegionTaskResult>> regionMap =</span> |
| <span class="source-line-no">322</span><span id="line-322"> new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">323</span><span id="line-323"> private ConcurrentMap<ServerName, LongAdder> perServerFailuresCount = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">324</span><span id="line-324"> private ConcurrentMap<String, LongAdder> perTableFailuresCount = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">325</span><span id="line-325"></span> |
| <span class="source-line-no">326</span><span id="line-326"> public ConcurrentMap<ServerName, LongAdder> getPerServerFailuresCount() {</span> |
| <span class="source-line-no">327</span><span id="line-327"> return perServerFailuresCount;</span> |
| <span class="source-line-no">328</span><span id="line-328"> }</span> |
| <span class="source-line-no">329</span><span id="line-329"></span> |
| <span class="source-line-no">330</span><span id="line-330"> public ConcurrentMap<String, LongAdder> getPerTableFailuresCount() {</span> |
| <span class="source-line-no">331</span><span id="line-331"> return perTableFailuresCount;</span> |
| <span class="source-line-no">332</span><span id="line-332"> }</span> |
| <span class="source-line-no">333</span><span id="line-333"></span> |
| <span class="source-line-no">334</span><span id="line-334"> public void resetFailuresCountDetails() {</span> |
| <span class="source-line-no">335</span><span id="line-335"> perServerFailuresCount.clear();</span> |
| <span class="source-line-no">336</span><span id="line-336"> perTableFailuresCount.clear();</span> |
| <span class="source-line-no">337</span><span id="line-337"> }</span> |
| <span class="source-line-no">338</span><span id="line-338"></span> |
| <span class="source-line-no">339</span><span id="line-339"> private void incFailuresCountDetails(ServerName serverName, RegionInfo region) {</span> |
| <span class="source-line-no">340</span><span id="line-340"> if (serverName != null) {</span> |
| <span class="source-line-no">341</span><span id="line-341"> perServerFailuresCount.compute(serverName, (server, count) -> {</span> |
| <span class="source-line-no">342</span><span id="line-342"> if (count == null) {</span> |
| <span class="source-line-no">343</span><span id="line-343"> count = new LongAdder();</span> |
| <span class="source-line-no">344</span><span id="line-344"> }</span> |
| <span class="source-line-no">345</span><span id="line-345"> count.increment();</span> |
| <span class="source-line-no">346</span><span id="line-346"> return count;</span> |
| <span class="source-line-no">347</span><span id="line-347"> });</span> |
| <span class="source-line-no">348</span><span id="line-348"> }</span> |
| <span class="source-line-no">349</span><span id="line-349"> perTableFailuresCount.compute(region.getTable().getNameAsString(), (tableName, count) -> {</span> |
| <span class="source-line-no">350</span><span id="line-350"> if (count == null) {</span> |
| <span class="source-line-no">351</span><span id="line-351"> count = new LongAdder();</span> |
| <span class="source-line-no">352</span><span id="line-352"> }</span> |
| <span class="source-line-no">353</span><span id="line-353"> count.increment();</span> |
| <span class="source-line-no">354</span><span id="line-354"> return count;</span> |
| <span class="source-line-no">355</span><span id="line-355"> });</span> |
| <span class="source-line-no">356</span><span id="line-356"> }</span> |
| <span class="source-line-no">357</span><span id="line-357"></span> |
| <span class="source-line-no">358</span><span id="line-358"> public void publishReadFailure(ServerName serverName, RegionInfo region, Exception e) {</span> |
| <span class="source-line-no">359</span><span id="line-359"> LOG.error("Read from {} on serverName={} failed", region.getRegionNameAsString(), serverName,</span> |
| <span class="source-line-no">360</span><span id="line-360"> e);</span> |
| <span class="source-line-no">361</span><span id="line-361"> incReadFailureCount();</span> |
| <span class="source-line-no">362</span><span id="line-362"> incFailuresCountDetails(serverName, region);</span> |
| <span class="source-line-no">363</span><span id="line-363"> }</span> |
| <span class="source-line-no">364</span><span id="line-364"></span> |
| <span class="source-line-no">365</span><span id="line-365"> public void publishReadFailure(ServerName serverName, RegionInfo region,</span> |
| <span class="source-line-no">366</span><span id="line-366"> ColumnFamilyDescriptor column, Exception e) {</span> |
| <span class="source-line-no">367</span><span id="line-367"> LOG.error("Read from {} on serverName={}, columnFamily={} failed",</span> |
| <span class="source-line-no">368</span><span id="line-368"> region.getRegionNameAsString(), serverName, column.getNameAsString(), e);</span> |
| <span class="source-line-no">369</span><span id="line-369"> incReadFailureCount();</span> |
| <span class="source-line-no">370</span><span id="line-370"> incFailuresCountDetails(serverName, region);</span> |
| <span class="source-line-no">371</span><span id="line-371"> }</span> |
| <span class="source-line-no">372</span><span id="line-372"></span> |
| <span class="source-line-no">373</span><span id="line-373"> public void publishReadTiming(ServerName serverName, RegionInfo region,</span> |
| <span class="source-line-no">374</span><span id="line-374"> ColumnFamilyDescriptor column, long msTime) {</span> |
| <span class="source-line-no">375</span><span id="line-375"> RegionTaskResult rtr = new RegionTaskResult(region, region.getTable(), serverName, column);</span> |
| <span class="source-line-no">376</span><span id="line-376"> rtr.setReadSuccess();</span> |
| <span class="source-line-no">377</span><span id="line-377"> rtr.setReadLatency(msTime);</span> |
| <span class="source-line-no">378</span><span id="line-378"> List<RegionTaskResult> rtrs = regionMap.get(region.getRegionNameAsString());</span> |
| <span class="source-line-no">379</span><span id="line-379"> rtrs.add(rtr);</span> |
| <span class="source-line-no">380</span><span id="line-380"> // Note that read success count will be equal to total column family read successes.</span> |
| <span class="source-line-no">381</span><span id="line-381"> incReadSuccessCount();</span> |
| <span class="source-line-no">382</span><span id="line-382"> LOG.info("Read from {} on {} {} in {}ms", region.getRegionNameAsString(), serverName,</span> |
| <span class="source-line-no">383</span><span id="line-383"> column.getNameAsString(), msTime);</span> |
| <span class="source-line-no">384</span><span id="line-384"> }</span> |
| <span class="source-line-no">385</span><span id="line-385"></span> |
| <span class="source-line-no">386</span><span id="line-386"> public void publishWriteFailure(ServerName serverName, RegionInfo region, Exception e) {</span> |
| <span class="source-line-no">387</span><span id="line-387"> LOG.error("Write to {} on {} failed", region.getRegionNameAsString(), serverName, e);</span> |
| <span class="source-line-no">388</span><span id="line-388"> incWriteFailureCount();</span> |
| <span class="source-line-no">389</span><span id="line-389"> incFailuresCountDetails(serverName, region);</span> |
| <span class="source-line-no">390</span><span id="line-390"> }</span> |
| <span class="source-line-no">391</span><span id="line-391"></span> |
| <span class="source-line-no">392</span><span id="line-392"> public void publishWriteFailure(ServerName serverName, RegionInfo region,</span> |
| <span class="source-line-no">393</span><span id="line-393"> ColumnFamilyDescriptor column, Exception e) {</span> |
| <span class="source-line-no">394</span><span id="line-394"> LOG.error("Write to {} on {} {} failed", region.getRegionNameAsString(), serverName,</span> |
| <span class="source-line-no">395</span><span id="line-395"> column.getNameAsString(), e);</span> |
| <span class="source-line-no">396</span><span id="line-396"> incWriteFailureCount();</span> |
| <span class="source-line-no">397</span><span id="line-397"> incFailuresCountDetails(serverName, region);</span> |
| <span class="source-line-no">398</span><span id="line-398"> }</span> |
| <span class="source-line-no">399</span><span id="line-399"></span> |
| <span class="source-line-no">400</span><span id="line-400"> public void publishWriteTiming(ServerName serverName, RegionInfo region,</span> |
| <span class="source-line-no">401</span><span id="line-401"> ColumnFamilyDescriptor column, long msTime) {</span> |
| <span class="source-line-no">402</span><span id="line-402"> RegionTaskResult rtr = new RegionTaskResult(region, region.getTable(), serverName, column);</span> |
| <span class="source-line-no">403</span><span id="line-403"> rtr.setWriteSuccess();</span> |
| <span class="source-line-no">404</span><span id="line-404"> rtr.setWriteLatency(msTime);</span> |
| <span class="source-line-no">405</span><span id="line-405"> List<RegionTaskResult> rtrs = regionMap.get(region.getRegionNameAsString());</span> |
| <span class="source-line-no">406</span><span id="line-406"> rtrs.add(rtr);</span> |
| <span class="source-line-no">407</span><span id="line-407"> // Note that write success count will be equal to total column family write successes.</span> |
| <span class="source-line-no">408</span><span id="line-408"> incWriteSuccessCount();</span> |
| <span class="source-line-no">409</span><span id="line-409"> LOG.info("Write to {} on {} {} in {}ms", region.getRegionNameAsString(), serverName,</span> |
| <span class="source-line-no">410</span><span id="line-410"> column.getNameAsString(), msTime);</span> |
| <span class="source-line-no">411</span><span id="line-411"> }</span> |
| <span class="source-line-no">412</span><span id="line-412"></span> |
| <span class="source-line-no">413</span><span id="line-413"> public Map<String, LongAdder> getReadLatencyMap() {</span> |
| <span class="source-line-no">414</span><span id="line-414"> return this.perTableReadLatency;</span> |
| <span class="source-line-no">415</span><span id="line-415"> }</span> |
| <span class="source-line-no">416</span><span id="line-416"></span> |
| <span class="source-line-no">417</span><span id="line-417"> public LongAdder initializeAndGetReadLatencyForTable(String tableName) {</span> |
| <span class="source-line-no">418</span><span id="line-418"> LongAdder initLatency = new LongAdder();</span> |
| <span class="source-line-no">419</span><span id="line-419"> this.perTableReadLatency.put(tableName, initLatency);</span> |
| <span class="source-line-no">420</span><span id="line-420"> return initLatency;</span> |
| <span class="source-line-no">421</span><span id="line-421"> }</span> |
| <span class="source-line-no">422</span><span id="line-422"></span> |
| <span class="source-line-no">423</span><span id="line-423"> public void initializeWriteLatency() {</span> |
| <span class="source-line-no">424</span><span id="line-424"> this.writeLatency.reset();</span> |
| <span class="source-line-no">425</span><span id="line-425"> }</span> |
| <span class="source-line-no">426</span><span id="line-426"></span> |
| <span class="source-line-no">427</span><span id="line-427"> public LongAdder getWriteLatency() {</span> |
| <span class="source-line-no">428</span><span id="line-428"> return this.writeLatency;</span> |
| <span class="source-line-no">429</span><span id="line-429"> }</span> |
| <span class="source-line-no">430</span><span id="line-430"></span> |
| <span class="source-line-no">431</span><span id="line-431"> public ConcurrentMap<String, List<RegionTaskResult>> getRegionMap() {</span> |
| <span class="source-line-no">432</span><span id="line-432"> return this.regionMap;</span> |
| <span class="source-line-no">433</span><span id="line-433"> }</span> |
| <span class="source-line-no">434</span><span id="line-434"></span> |
| <span class="source-line-no">435</span><span id="line-435"> public int getTotalExpectedRegions() {</span> |
| <span class="source-line-no">436</span><span id="line-436"> return this.regionMap.size();</span> |
| <span class="source-line-no">437</span><span id="line-437"> }</span> |
| <span class="source-line-no">438</span><span id="line-438"> }</span> |
| <span class="source-line-no">439</span><span id="line-439"></span> |
| <span class="source-line-no">440</span><span id="line-440"> /**</span> |
| <span class="source-line-no">441</span><span id="line-441"> * Run a single zookeeper Task and then exit.</span> |
| <span class="source-line-no">442</span><span id="line-442"> */</span> |
| <span class="source-line-no">443</span><span id="line-443"> static class ZookeeperTask implements Callable<Void> {</span> |
| <span class="source-line-no">444</span><span id="line-444"> private final Connection connection;</span> |
| <span class="source-line-no">445</span><span id="line-445"> private final String host;</span> |
| <span class="source-line-no">446</span><span id="line-446"> private String znode;</span> |
| <span class="source-line-no">447</span><span id="line-447"> private final int timeout;</span> |
| <span class="source-line-no">448</span><span id="line-448"> private ZookeeperStdOutSink sink;</span> |
| <span class="source-line-no">449</span><span id="line-449"></span> |
| <span class="source-line-no">450</span><span id="line-450"> public ZookeeperTask(Connection connection, String host, String znode, int timeout,</span> |
| <span class="source-line-no">451</span><span id="line-451"> ZookeeperStdOutSink sink) {</span> |
| <span class="source-line-no">452</span><span id="line-452"> this.connection = connection;</span> |
| <span class="source-line-no">453</span><span id="line-453"> this.host = host;</span> |
| <span class="source-line-no">454</span><span id="line-454"> this.znode = znode;</span> |
| <span class="source-line-no">455</span><span id="line-455"> this.timeout = timeout;</span> |
| <span class="source-line-no">456</span><span id="line-456"> this.sink = sink;</span> |
| <span class="source-line-no">457</span><span id="line-457"> }</span> |
| <span class="source-line-no">458</span><span id="line-458"></span> |
| <span class="source-line-no">459</span><span id="line-459"> @Override</span> |
| <span class="source-line-no">460</span><span id="line-460"> public Void call() throws Exception {</span> |
| <span class="source-line-no">461</span><span id="line-461"> if (this.sink.isStopped()) {</span> |
| <span class="source-line-no">462</span><span id="line-462"> return null;</span> |
| <span class="source-line-no">463</span><span id="line-463"> }</span> |
| <span class="source-line-no">464</span><span id="line-464"> ZooKeeper zooKeeper = null;</span> |
| <span class="source-line-no">465</span><span id="line-465"> try {</span> |
| <span class="source-line-no">466</span><span id="line-466"> zooKeeper = new ZooKeeper(host, timeout, EmptyWatcher.instance);</span> |
| <span class="source-line-no">467</span><span id="line-467"> Stat exists = zooKeeper.exists(znode, false);</span> |
| <span class="source-line-no">468</span><span id="line-468"> StopWatch stopwatch = new StopWatch();</span> |
| <span class="source-line-no">469</span><span id="line-469"> stopwatch.start();</span> |
| <span class="source-line-no">470</span><span id="line-470"> zooKeeper.getData(znode, false, exists);</span> |
| <span class="source-line-no">471</span><span id="line-471"> stopwatch.stop();</span> |
| <span class="source-line-no">472</span><span id="line-472"> sink.publishReadTiming(znode, host, stopwatch.getTime());</span> |
| <span class="source-line-no">473</span><span id="line-473"> } catch (KeeperException | InterruptedException e) {</span> |
| <span class="source-line-no">474</span><span id="line-474"> sink.publishReadFailure(znode, host);</span> |
| <span class="source-line-no">475</span><span id="line-475"> } finally {</span> |
| <span class="source-line-no">476</span><span id="line-476"> if (zooKeeper != null) {</span> |
| <span class="source-line-no">477</span><span id="line-477"> zooKeeper.close();</span> |
| <span class="source-line-no">478</span><span id="line-478"> }</span> |
| <span class="source-line-no">479</span><span id="line-479"> }</span> |
| <span class="source-line-no">480</span><span id="line-480"> return null;</span> |
| <span class="source-line-no">481</span><span id="line-481"> }</span> |
| <span class="source-line-no">482</span><span id="line-482"> }</span> |
| <span class="source-line-no">483</span><span id="line-483"></span> |
| <span class="source-line-no">484</span><span id="line-484"> /**</span> |
| <span class="source-line-no">485</span><span id="line-485"> * Run a single Region Task and then exit. For each column family of the Region, get one row and</span> |
| <span class="source-line-no">486</span><span id="line-486"> * output latency or failure.</span> |
| <span class="source-line-no">487</span><span id="line-487"> */</span> |
| <span class="source-line-no">488</span><span id="line-488"> static class RegionTask implements Callable<Void> {</span> |
| <span class="source-line-no">489</span><span id="line-489"> public enum TaskType {</span> |
| <span class="source-line-no">490</span><span id="line-490"> READ,</span> |
| <span class="source-line-no">491</span><span id="line-491"> WRITE</span> |
| <span class="source-line-no">492</span><span id="line-492"> }</span> |
| <span class="source-line-no">493</span><span id="line-493"></span> |
| <span class="source-line-no">494</span><span id="line-494"> private Connection connection;</span> |
| <span class="source-line-no">495</span><span id="line-495"> private RegionInfo region;</span> |
| <span class="source-line-no">496</span><span id="line-496"> private RegionStdOutSink sink;</span> |
| <span class="source-line-no">497</span><span id="line-497"> private TaskType taskType;</span> |
| <span class="source-line-no">498</span><span id="line-498"> private boolean rawScanEnabled;</span> |
| <span class="source-line-no">499</span><span id="line-499"> private ServerName serverName;</span> |
| <span class="source-line-no">500</span><span id="line-500"> private LongAdder readWriteLatency;</span> |
| <span class="source-line-no">501</span><span id="line-501"> private boolean readAllCF;</span> |
| <span class="source-line-no">502</span><span id="line-502"></span> |
| <span class="source-line-no">503</span><span id="line-503"> RegionTask(Connection connection, RegionInfo region, ServerName serverName,</span> |
| <span class="source-line-no">504</span><span id="line-504"> RegionStdOutSink sink, TaskType taskType, boolean rawScanEnabled, LongAdder rwLatency,</span> |
| <span class="source-line-no">505</span><span id="line-505"> boolean readAllCF) {</span> |
| <span class="source-line-no">506</span><span id="line-506"> this.connection = connection;</span> |
| <span class="source-line-no">507</span><span id="line-507"> this.region = region;</span> |
| <span class="source-line-no">508</span><span id="line-508"> this.serverName = serverName;</span> |
| <span class="source-line-no">509</span><span id="line-509"> this.sink = sink;</span> |
| <span class="source-line-no">510</span><span id="line-510"> this.taskType = taskType;</span> |
| <span class="source-line-no">511</span><span id="line-511"> this.rawScanEnabled = rawScanEnabled;</span> |
| <span class="source-line-no">512</span><span id="line-512"> this.readWriteLatency = rwLatency;</span> |
| <span class="source-line-no">513</span><span id="line-513"> this.readAllCF = readAllCF;</span> |
| <span class="source-line-no">514</span><span id="line-514"> }</span> |
| <span class="source-line-no">515</span><span id="line-515"></span> |
| <span class="source-line-no">516</span><span id="line-516"> @Override</span> |
| <span class="source-line-no">517</span><span id="line-517"> public Void call() {</span> |
| <span class="source-line-no">518</span><span id="line-518"> if (this.sink.isStopped()) {</span> |
| <span class="source-line-no">519</span><span id="line-519"> return null;</span> |
| <span class="source-line-no">520</span><span id="line-520"> }</span> |
| <span class="source-line-no">521</span><span id="line-521"> switch (taskType) {</span> |
| <span class="source-line-no">522</span><span id="line-522"> case READ:</span> |
| <span class="source-line-no">523</span><span id="line-523"> return read();</span> |
| <span class="source-line-no">524</span><span id="line-524"> case WRITE:</span> |
| <span class="source-line-no">525</span><span id="line-525"> return write();</span> |
| <span class="source-line-no">526</span><span id="line-526"> default:</span> |
| <span class="source-line-no">527</span><span id="line-527"> return read();</span> |
| <span class="source-line-no">528</span><span id="line-528"> }</span> |
| <span class="source-line-no">529</span><span id="line-529"> }</span> |
| <span class="source-line-no">530</span><span id="line-530"></span> |
| <span class="source-line-no">531</span><span id="line-531"> private Void readColumnFamily(Table table, ColumnFamilyDescriptor column) {</span> |
| <span class="source-line-no">532</span><span id="line-532"> byte[] startKey = null;</span> |
| <span class="source-line-no">533</span><span id="line-533"> Scan scan = null;</span> |
| <span class="source-line-no">534</span><span id="line-534"> ResultScanner rs = null;</span> |
| <span class="source-line-no">535</span><span id="line-535"> StopWatch stopWatch = new StopWatch();</span> |
| <span class="source-line-no">536</span><span id="line-536"> startKey = region.getStartKey();</span> |
| <span class="source-line-no">537</span><span id="line-537"> // Can't do a get on empty start row so do a Scan of first element if any instead.</span> |
| <span class="source-line-no">538</span><span id="line-538"> if (startKey.length > 0) {</span> |
| <span class="source-line-no">539</span><span id="line-539"> Get get = new Get(startKey);</span> |
| <span class="source-line-no">540</span><span id="line-540"> get.setCacheBlocks(false);</span> |
| <span class="source-line-no">541</span><span id="line-541"> get.setFilter(new FirstKeyOnlyFilter());</span> |
| <span class="source-line-no">542</span><span id="line-542"> get.addFamily(column.getName());</span> |
| <span class="source-line-no">543</span><span id="line-543"> // Converting get object to scan to enable RAW SCAN.</span> |
| <span class="source-line-no">544</span><span id="line-544"> // This will work for all the regions of the HBase tables except first region of the table.</span> |
| <span class="source-line-no">545</span><span id="line-545"> scan = new Scan(get);</span> |
| <span class="source-line-no">546</span><span id="line-546"> scan.setRaw(rawScanEnabled);</span> |
| <span class="source-line-no">547</span><span id="line-547"> } else {</span> |
| <span class="source-line-no">548</span><span id="line-548"> scan = new Scan();</span> |
| <span class="source-line-no">549</span><span id="line-549"> // In case of first region of the HBase Table, we do not have start-key for the region.</span> |
| <span class="source-line-no">550</span><span id="line-550"> // For Region Canary, we only need to scan a single row/cell in the region to make sure that</span> |
| <span class="source-line-no">551</span><span id="line-551"> // region is accessible.</span> |
| <span class="source-line-no">552</span><span id="line-552"> //</span> |
| <span class="source-line-no">553</span><span id="line-553"> // When HBase table has more than 1 empty regions at start of the row-key space, Canary will</span> |
| <span class="source-line-no">554</span><span id="line-554"> // create multiple scan object to find first available row in the table by scanning all the</span> |
| <span class="source-line-no">555</span><span id="line-555"> // regions in sequence until it can find first available row.</span> |
| <span class="source-line-no">556</span><span id="line-556"> //</span> |
| <span class="source-line-no">557</span><span id="line-557"> // This could result in multiple millions of scans based on the size of table and number of</span> |
| <span class="source-line-no">558</span><span id="line-558"> // empty regions in sequence. In test environment, A table with no data and 1100 empty</span> |
| <span class="source-line-no">559</span><span id="line-559"> // regions, Single canary run was creating close to half million to 1 million scans to</span> |
| <span class="source-line-no">560</span><span id="line-560"> // successfully do canary run for the table.</span> |
| <span class="source-line-no">561</span><span id="line-561"> //</span> |
| <span class="source-line-no">562</span><span id="line-562"> // Since First region of the table doesn't have any start key, We should set End Key as</span> |
| <span class="source-line-no">563</span><span id="line-563"> // stop row and set inclusive=false to limit scan to single region only.</span> |
| <span class="source-line-no">564</span><span id="line-564"> //</span> |
| <span class="source-line-no">565</span><span id="line-565"> // TODO : In future, we can streamline Canary behaviour for all the regions by doing scan</span> |
| <span class="source-line-no">566</span><span id="line-566"> // with startRow inclusive and stopRow exclusive instead of different behaviour for First</span> |
| <span class="source-line-no">567</span><span id="line-567"> // Region of the table and rest of the region of the table. This way implementation is</span> |
| <span class="source-line-no">568</span><span id="line-568"> // simplified. As of now this change has been kept minimal to avoid any unnecessary</span> |
| <span class="source-line-no">569</span><span id="line-569"> // perf impact.</span> |
| <span class="source-line-no">570</span><span id="line-570"> scan.withStopRow(region.getEndKey(), false);</span> |
| <span class="source-line-no">571</span><span id="line-571"> LOG.debug("rawScan {} for {}", rawScanEnabled, region.getTable());</span> |
| <span class="source-line-no">572</span><span id="line-572"> scan.setRaw(rawScanEnabled);</span> |
| <span class="source-line-no">573</span><span id="line-573"> scan.setCaching(1);</span> |
| <span class="source-line-no">574</span><span id="line-574"> scan.setCacheBlocks(false);</span> |
| <span class="source-line-no">575</span><span id="line-575"> scan.setFilter(new FirstKeyOnlyFilter());</span> |
| <span class="source-line-no">576</span><span id="line-576"> scan.addFamily(column.getName());</span> |
| <span class="source-line-no">577</span><span id="line-577"> scan.setMaxResultSize(1L);</span> |
| <span class="source-line-no">578</span><span id="line-578"> scan.setOneRowLimit();</span> |
| <span class="source-line-no">579</span><span id="line-579"> }</span> |
| <span class="source-line-no">580</span><span id="line-580"> LOG.debug("Reading from {} {} {} {}", region.getTable(), region.getRegionNameAsString(),</span> |
| <span class="source-line-no">581</span><span id="line-581"> column.getNameAsString(), Bytes.toStringBinary(startKey));</span> |
| <span class="source-line-no">582</span><span id="line-582"> try {</span> |
| <span class="source-line-no">583</span><span id="line-583"> stopWatch.start();</span> |
| <span class="source-line-no">584</span><span id="line-584"> rs = table.getScanner(scan);</span> |
| <span class="source-line-no">585</span><span id="line-585"> rs.next();</span> |
| <span class="source-line-no">586</span><span id="line-586"> stopWatch.stop();</span> |
| <span class="source-line-no">587</span><span id="line-587"> this.readWriteLatency.add(stopWatch.getTime());</span> |
| <span class="source-line-no">588</span><span id="line-588"> sink.publishReadTiming(serverName, region, column, stopWatch.getTime());</span> |
| <span class="source-line-no">589</span><span id="line-589"> } catch (Exception e) {</span> |
| <span class="source-line-no">590</span><span id="line-590"> sink.publishReadFailure(serverName, region, column, e);</span> |
| <span class="source-line-no">591</span><span id="line-591"> sink.updateReadFailures(region.getRegionNameAsString(),</span> |
| <span class="source-line-no">592</span><span id="line-592"> serverName == null ? "NULL" : serverName.getHostname());</span> |
| <span class="source-line-no">593</span><span id="line-593"> } finally {</span> |
| <span class="source-line-no">594</span><span id="line-594"> if (rs != null) {</span> |
| <span class="source-line-no">595</span><span id="line-595"> rs.close();</span> |
| <span class="source-line-no">596</span><span id="line-596"> }</span> |
| <span class="source-line-no">597</span><span id="line-597"> }</span> |
| <span class="source-line-no">598</span><span id="line-598"> return null;</span> |
| <span class="source-line-no">599</span><span id="line-599"> }</span> |
| <span class="source-line-no">600</span><span id="line-600"></span> |
| <span class="source-line-no">601</span><span id="line-601"> private ColumnFamilyDescriptor randomPickOneColumnFamily(ColumnFamilyDescriptor[] cfs) {</span> |
| <span class="source-line-no">602</span><span id="line-602"> int size = cfs.length;</span> |
| <span class="source-line-no">603</span><span id="line-603"> return cfs[ThreadLocalRandom.current().nextInt(size)];</span> |
| <span class="source-line-no">604</span><span id="line-604"></span> |
| <span class="source-line-no">605</span><span id="line-605"> }</span> |
| <span class="source-line-no">606</span><span id="line-606"></span> |
| <span class="source-line-no">607</span><span id="line-607"> public Void read() {</span> |
| <span class="source-line-no">608</span><span id="line-608"> Table table = null;</span> |
| <span class="source-line-no">609</span><span id="line-609"> TableDescriptor tableDesc = null;</span> |
| <span class="source-line-no">610</span><span id="line-610"> try {</span> |
| <span class="source-line-no">611</span><span id="line-611"> LOG.debug("Reading table descriptor for table {}", region.getTable());</span> |
| <span class="source-line-no">612</span><span id="line-612"> table = connection.getTable(region.getTable());</span> |
| <span class="source-line-no">613</span><span id="line-613"> tableDesc = table.getDescriptor();</span> |
| <span class="source-line-no">614</span><span id="line-614"> } catch (IOException e) {</span> |
| <span class="source-line-no">615</span><span id="line-615"> LOG.debug("sniffRegion {} of {} failed", region.getEncodedName(), e);</span> |
| <span class="source-line-no">616</span><span id="line-616"> sink.publishReadFailure(serverName, region, e);</span> |
| <span class="source-line-no">617</span><span id="line-617"> if (table != null) {</span> |
| <span class="source-line-no">618</span><span id="line-618"> try {</span> |
| <span class="source-line-no">619</span><span id="line-619"> table.close();</span> |
| <span class="source-line-no">620</span><span id="line-620"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">621</span><span id="line-621"> LOG.error("Close table failed", e);</span> |
| <span class="source-line-no">622</span><span id="line-622"> }</span> |
| <span class="source-line-no">623</span><span id="line-623"> }</span> |
| <span class="source-line-no">624</span><span id="line-624"> return null;</span> |
| <span class="source-line-no">625</span><span id="line-625"> }</span> |
| <span class="source-line-no">626</span><span id="line-626"></span> |
| <span class="source-line-no">627</span><span id="line-627"> if (readAllCF) {</span> |
| <span class="source-line-no">628</span><span id="line-628"> for (ColumnFamilyDescriptor column : tableDesc.getColumnFamilies()) {</span> |
| <span class="source-line-no">629</span><span id="line-629"> readColumnFamily(table, column);</span> |
| <span class="source-line-no">630</span><span id="line-630"> }</span> |
| <span class="source-line-no">631</span><span id="line-631"> } else {</span> |
| <span class="source-line-no">632</span><span id="line-632"> readColumnFamily(table, randomPickOneColumnFamily(tableDesc.getColumnFamilies()));</span> |
| <span class="source-line-no">633</span><span id="line-633"> }</span> |
| <span class="source-line-no">634</span><span id="line-634"> try {</span> |
| <span class="source-line-no">635</span><span id="line-635"> table.close();</span> |
| <span class="source-line-no">636</span><span id="line-636"> } catch (IOException e) {</span> |
| <span class="source-line-no">637</span><span id="line-637"> LOG.error("Close table failed", e);</span> |
| <span class="source-line-no">638</span><span id="line-638"> }</span> |
| <span class="source-line-no">639</span><span id="line-639"> return null;</span> |
| <span class="source-line-no">640</span><span id="line-640"> }</span> |
| <span class="source-line-no">641</span><span id="line-641"></span> |
| <span class="source-line-no">642</span><span id="line-642"> /**</span> |
| <span class="source-line-no">643</span><span id="line-643"> * Check writes for the canary table</span> |
| <span class="source-line-no">644</span><span id="line-644"> */</span> |
| <span class="source-line-no">645</span><span id="line-645"> private Void write() {</span> |
| <span class="source-line-no">646</span><span id="line-646"> Table table = null;</span> |
| <span class="source-line-no">647</span><span id="line-647"> TableDescriptor tableDesc = null;</span> |
| <span class="source-line-no">648</span><span id="line-648"> try {</span> |
| <span class="source-line-no">649</span><span id="line-649"> table = connection.getTable(region.getTable());</span> |
| <span class="source-line-no">650</span><span id="line-650"> tableDesc = table.getDescriptor();</span> |
| <span class="source-line-no">651</span><span id="line-651"> byte[] rowToCheck = region.getStartKey();</span> |
| <span class="source-line-no">652</span><span id="line-652"> if (rowToCheck.length == 0) {</span> |
| <span class="source-line-no">653</span><span id="line-653"> rowToCheck = new byte[] { 0x0 };</span> |
| <span class="source-line-no">654</span><span id="line-654"> }</span> |
| <span class="source-line-no">655</span><span id="line-655"> int writeValueSize =</span> |
| <span class="source-line-no">656</span><span id="line-656"> connection.getConfiguration().getInt(HConstants.HBASE_CANARY_WRITE_VALUE_SIZE_KEY, 10);</span> |
| <span class="source-line-no">657</span><span id="line-657"> for (ColumnFamilyDescriptor column : tableDesc.getColumnFamilies()) {</span> |
| <span class="source-line-no">658</span><span id="line-658"> Put put = new Put(rowToCheck);</span> |
| <span class="source-line-no">659</span><span id="line-659"> byte[] value = new byte[writeValueSize];</span> |
| <span class="source-line-no">660</span><span id="line-660"> Bytes.random(value);</span> |
| <span class="source-line-no">661</span><span id="line-661"> put.addColumn(column.getName(), HConstants.EMPTY_BYTE_ARRAY, value);</span> |
| <span class="source-line-no">662</span><span id="line-662"> LOG.debug("Writing to {} {} {} {}", tableDesc.getTableName(),</span> |
| <span class="source-line-no">663</span><span id="line-663"> region.getRegionNameAsString(), column.getNameAsString(),</span> |
| <span class="source-line-no">664</span><span id="line-664"> Bytes.toStringBinary(rowToCheck));</span> |
| <span class="source-line-no">665</span><span id="line-665"> try {</span> |
| <span class="source-line-no">666</span><span id="line-666"> long startTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">667</span><span id="line-667"> table.put(put);</span> |
| <span class="source-line-no">668</span><span id="line-668"> long time = EnvironmentEdgeManager.currentTime() - startTime;</span> |
| <span class="source-line-no">669</span><span id="line-669"> this.readWriteLatency.add(time);</span> |
| <span class="source-line-no">670</span><span id="line-670"> sink.publishWriteTiming(serverName, region, column, time);</span> |
| <span class="source-line-no">671</span><span id="line-671"> } catch (Exception e) {</span> |
| <span class="source-line-no">672</span><span id="line-672"> sink.publishWriteFailure(serverName, region, column, e);</span> |
| <span class="source-line-no">673</span><span id="line-673"> }</span> |
| <span class="source-line-no">674</span><span id="line-674"> }</span> |
| <span class="source-line-no">675</span><span id="line-675"> table.close();</span> |
| <span class="source-line-no">676</span><span id="line-676"> } catch (IOException e) {</span> |
| <span class="source-line-no">677</span><span id="line-677"> sink.publishWriteFailure(serverName, region, e);</span> |
| <span class="source-line-no">678</span><span id="line-678"> sink.updateWriteFailures(region.getRegionNameAsString(), serverName.getHostname());</span> |
| <span class="source-line-no">679</span><span id="line-679"> }</span> |
| <span class="source-line-no">680</span><span id="line-680"> return null;</span> |
| <span class="source-line-no">681</span><span id="line-681"> }</span> |
| <span class="source-line-no">682</span><span id="line-682"> }</span> |
| <span class="source-line-no">683</span><span id="line-683"></span> |
| <span class="source-line-no">684</span><span id="line-684"> /**</span> |
| <span class="source-line-no">685</span><span id="line-685"> * Run a single RegionServer Task and then exit. Get one row from a region on the regionserver and</span> |
| <span class="source-line-no">686</span><span id="line-686"> * output latency or the failure.</span> |
| <span class="source-line-no">687</span><span id="line-687"> */</span> |
| <span class="source-line-no">688</span><span id="line-688"> static class RegionServerTask implements Callable<Void> {</span> |
| <span class="source-line-no">689</span><span id="line-689"> private Connection connection;</span> |
| <span class="source-line-no">690</span><span id="line-690"> private String serverName;</span> |
| <span class="source-line-no">691</span><span id="line-691"> private RegionInfo region;</span> |
| <span class="source-line-no">692</span><span id="line-692"> private RegionServerStdOutSink sink;</span> |
| <span class="source-line-no">693</span><span id="line-693"> private Boolean rawScanEnabled;</span> |
| <span class="source-line-no">694</span><span id="line-694"> private AtomicLong successes;</span> |
| <span class="source-line-no">695</span><span id="line-695"></span> |
| <span class="source-line-no">696</span><span id="line-696"> RegionServerTask(Connection connection, String serverName, RegionInfo region,</span> |
| <span class="source-line-no">697</span><span id="line-697"> RegionServerStdOutSink sink, Boolean rawScanEnabled, AtomicLong successes) {</span> |
| <span class="source-line-no">698</span><span id="line-698"> this.connection = connection;</span> |
| <span class="source-line-no">699</span><span id="line-699"> this.serverName = serverName;</span> |
| <span class="source-line-no">700</span><span id="line-700"> this.region = region;</span> |
| <span class="source-line-no">701</span><span id="line-701"> this.sink = sink;</span> |
| <span class="source-line-no">702</span><span id="line-702"> this.rawScanEnabled = rawScanEnabled;</span> |
| <span class="source-line-no">703</span><span id="line-703"> this.successes = successes;</span> |
| <span class="source-line-no">704</span><span id="line-704"> }</span> |
| <span class="source-line-no">705</span><span id="line-705"></span> |
| <span class="source-line-no">706</span><span id="line-706"> @Override</span> |
| <span class="source-line-no">707</span><span id="line-707"> public Void call() {</span> |
| <span class="source-line-no">708</span><span id="line-708"> if (this.sink.isStopped()) {</span> |
| <span class="source-line-no">709</span><span id="line-709"> return null;</span> |
| <span class="source-line-no">710</span><span id="line-710"> }</span> |
| <span class="source-line-no">711</span><span id="line-711"> TableName tableName = null;</span> |
| <span class="source-line-no">712</span><span id="line-712"> Table table = null;</span> |
| <span class="source-line-no">713</span><span id="line-713"> Get get = null;</span> |
| <span class="source-line-no">714</span><span id="line-714"> byte[] startKey = null;</span> |
| <span class="source-line-no">715</span><span id="line-715"> Scan scan = null;</span> |
| <span class="source-line-no">716</span><span id="line-716"> StopWatch stopWatch = new StopWatch();</span> |
| <span class="source-line-no">717</span><span id="line-717"> // monitor one region on every region server</span> |
| <span class="source-line-no">718</span><span id="line-718"> stopWatch.reset();</span> |
| <span class="source-line-no">719</span><span id="line-719"> try {</span> |
| <span class="source-line-no">720</span><span id="line-720"> tableName = region.getTable();</span> |
| <span class="source-line-no">721</span><span id="line-721"> table = connection.getTable(tableName);</span> |
| <span class="source-line-no">722</span><span id="line-722"> startKey = region.getStartKey();</span> |
| <span class="source-line-no">723</span><span id="line-723"> // Can't do a get on empty start row so do a Scan of first element if any instead.</span> |
| <span class="source-line-no">724</span><span id="line-724"> LOG.debug("Reading from {} {} {} {}", serverName, region.getTable(),</span> |
| <span class="source-line-no">725</span><span id="line-725"> region.getRegionNameAsString(), Bytes.toStringBinary(startKey));</span> |
| <span class="source-line-no">726</span><span id="line-726"> if (startKey.length > 0) {</span> |
| <span class="source-line-no">727</span><span id="line-727"> get = new Get(startKey);</span> |
| <span class="source-line-no">728</span><span id="line-728"> get.setCacheBlocks(false);</span> |
| <span class="source-line-no">729</span><span id="line-729"> get.setFilter(new FirstKeyOnlyFilter());</span> |
| <span class="source-line-no">730</span><span id="line-730"> // Converting get object to scan to enable RAW SCAN.</span> |
| <span class="source-line-no">731</span><span id="line-731"> // This will work for all the regions of the HBase tables except first region.</span> |
| <span class="source-line-no">732</span><span id="line-732"> scan = new Scan(get);</span> |
| <span class="source-line-no">733</span><span id="line-733"></span> |
| <span class="source-line-no">734</span><span id="line-734"> } else {</span> |
| <span class="source-line-no">735</span><span id="line-735"> scan = new Scan();</span> |
| <span class="source-line-no">736</span><span id="line-736"> // In case of first region of the HBase Table, we do not have start-key for the region.</span> |
| <span class="source-line-no">737</span><span id="line-737"> // For Region Canary, we only need scan a single row/cell in the region to make sure that</span> |
| <span class="source-line-no">738</span><span id="line-738"> // region is accessible.</span> |
| <span class="source-line-no">739</span><span id="line-739"> //</span> |
| <span class="source-line-no">740</span><span id="line-740"> // When HBase table has more than 1 empty regions at start of the row-key space, Canary</span> |
| <span class="source-line-no">741</span><span id="line-741"> // will create multiple scan object to find first available row in the table by scanning</span> |
| <span class="source-line-no">742</span><span id="line-742"> // all the regions in sequence until it can find first available row.</span> |
| <span class="source-line-no">743</span><span id="line-743"> //</span> |
| <span class="source-line-no">744</span><span id="line-744"> // Since First region of the table doesn't have any start key, We should set End Key as</span> |
| <span class="source-line-no">745</span><span id="line-745"> // stop row and set inclusive=false to limit scan to first region only.</span> |
| <span class="source-line-no">746</span><span id="line-746"> scan.withStopRow(region.getEndKey(), false);</span> |
| <span class="source-line-no">747</span><span id="line-747"> scan.setCacheBlocks(false);</span> |
| <span class="source-line-no">748</span><span id="line-748"> scan.setFilter(new FirstKeyOnlyFilter());</span> |
| <span class="source-line-no">749</span><span id="line-749"> scan.setCaching(1);</span> |
| <span class="source-line-no">750</span><span id="line-750"> scan.setMaxResultSize(1L);</span> |
| <span class="source-line-no">751</span><span id="line-751"> scan.setOneRowLimit();</span> |
| <span class="source-line-no">752</span><span id="line-752"> }</span> |
| <span class="source-line-no">753</span><span id="line-753"> scan.setRaw(rawScanEnabled);</span> |
| <span class="source-line-no">754</span><span id="line-754"> stopWatch.start();</span> |
| <span class="source-line-no">755</span><span id="line-755"> ResultScanner s = table.getScanner(scan);</span> |
| <span class="source-line-no">756</span><span id="line-756"> s.next();</span> |
| <span class="source-line-no">757</span><span id="line-757"> s.close();</span> |
| <span class="source-line-no">758</span><span id="line-758"> stopWatch.stop();</span> |
| <span class="source-line-no">759</span><span id="line-759"> successes.incrementAndGet();</span> |
| <span class="source-line-no">760</span><span id="line-760"> sink.publishReadTiming(tableName.getNameAsString(), serverName, stopWatch.getTime());</span> |
| <span class="source-line-no">761</span><span id="line-761"> } catch (TableNotFoundException tnfe) {</span> |
| <span class="source-line-no">762</span><span id="line-762"> LOG.error("Table may be deleted", tnfe);</span> |
| <span class="source-line-no">763</span><span id="line-763"> // This is ignored because it doesn't imply that the regionserver is dead</span> |
| <span class="source-line-no">764</span><span id="line-764"> } catch (TableNotEnabledException tnee) {</span> |
| <span class="source-line-no">765</span><span id="line-765"> // This is considered a success since we got a response.</span> |
| <span class="source-line-no">766</span><span id="line-766"> successes.incrementAndGet();</span> |
| <span class="source-line-no">767</span><span id="line-767"> LOG.debug("The targeted table was disabled. Assuming success.");</span> |
| <span class="source-line-no">768</span><span id="line-768"> } catch (DoNotRetryIOException dnrioe) {</span> |
| <span class="source-line-no">769</span><span id="line-769"> sink.publishReadFailure(tableName.getNameAsString(), serverName);</span> |
| <span class="source-line-no">770</span><span id="line-770"> LOG.error(dnrioe.toString(), dnrioe);</span> |
| <span class="source-line-no">771</span><span id="line-771"> } catch (IOException e) {</span> |
| <span class="source-line-no">772</span><span id="line-772"> sink.publishReadFailure(tableName.getNameAsString(), serverName);</span> |
| <span class="source-line-no">773</span><span id="line-773"> LOG.error(e.toString(), e);</span> |
| <span class="source-line-no">774</span><span id="line-774"> } finally {</span> |
| <span class="source-line-no">775</span><span id="line-775"> if (table != null) {</span> |
| <span class="source-line-no">776</span><span id="line-776"> try {</span> |
| <span class="source-line-no">777</span><span id="line-777"> table.close();</span> |
| <span class="source-line-no">778</span><span id="line-778"> } catch (IOException e) {/* DO NOTHING */</span> |
| <span class="source-line-no">779</span><span id="line-779"> LOG.error("Close table failed", e);</span> |
| <span class="source-line-no">780</span><span id="line-780"> }</span> |
| <span class="source-line-no">781</span><span id="line-781"> }</span> |
| <span class="source-line-no">782</span><span id="line-782"> scan = null;</span> |
| <span class="source-line-no">783</span><span id="line-783"> get = null;</span> |
| <span class="source-line-no">784</span><span id="line-784"> startKey = null;</span> |
| <span class="source-line-no">785</span><span id="line-785"> }</span> |
| <span class="source-line-no">786</span><span id="line-786"> return null;</span> |
| <span class="source-line-no">787</span><span id="line-787"> }</span> |
| <span class="source-line-no">788</span><span id="line-788"> }</span> |
| <span class="source-line-no">789</span><span id="line-789"></span> |
| <span class="source-line-no">790</span><span id="line-790"> private static final int USAGE_EXIT_CODE = 1;</span> |
| <span class="source-line-no">791</span><span id="line-791"> private static final int INIT_ERROR_EXIT_CODE = 2;</span> |
| <span class="source-line-no">792</span><span id="line-792"> private static final int TIMEOUT_ERROR_EXIT_CODE = 3;</span> |
| <span class="source-line-no">793</span><span id="line-793"> private static final int ERROR_EXIT_CODE = 4;</span> |
| <span class="source-line-no">794</span><span id="line-794"> private static final int FAILURE_EXIT_CODE = 5;</span> |
| <span class="source-line-no">795</span><span id="line-795"></span> |
| <span class="source-line-no">796</span><span id="line-796"> private static final long DEFAULT_INTERVAL = 60000;</span> |
| <span class="source-line-no">797</span><span id="line-797"></span> |
| <span class="source-line-no">798</span><span id="line-798"> private static final long DEFAULT_TIMEOUT = 600000; // 10 mins</span> |
| <span class="source-line-no">799</span><span id="line-799"> private static final int MAX_THREADS_NUM = 16; // #threads to contact regions</span> |
| <span class="source-line-no">800</span><span id="line-800"></span> |
| <span class="source-line-no">801</span><span id="line-801"> private static final Logger LOG = LoggerFactory.getLogger(Canary.class);</span> |
| <span class="source-line-no">802</span><span id="line-802"></span> |
| <span class="source-line-no">803</span><span id="line-803"> public static final TableName DEFAULT_WRITE_TABLE_NAME =</span> |
| <span class="source-line-no">804</span><span id="line-804"> TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "canary");</span> |
| <span class="source-line-no">805</span><span id="line-805"></span> |
| <span class="source-line-no">806</span><span id="line-806"> private static final String CANARY_TABLE_FAMILY_NAME = "Test";</span> |
| <span class="source-line-no">807</span><span id="line-807"></span> |
| <span class="source-line-no">808</span><span id="line-808"> private Configuration conf = null;</span> |
| <span class="source-line-no">809</span><span id="line-809"> private long interval = 0;</span> |
| <span class="source-line-no">810</span><span id="line-810"> private Sink sink = null;</span> |
| <span class="source-line-no">811</span><span id="line-811"></span> |
| <span class="source-line-no">812</span><span id="line-812"> /**</span> |
| <span class="source-line-no">813</span><span id="line-813"> * True if we are to run in 'regionServer' mode.</span> |
| <span class="source-line-no">814</span><span id="line-814"> */</span> |
| <span class="source-line-no">815</span><span id="line-815"> private boolean regionServerMode = false;</span> |
| <span class="source-line-no">816</span><span id="line-816"></span> |
| <span class="source-line-no">817</span><span id="line-817"> /**</span> |
| <span class="source-line-no">818</span><span id="line-818"> * True if we are to run in zookeeper 'mode'.</span> |
| <span class="source-line-no">819</span><span id="line-819"> */</span> |
| <span class="source-line-no">820</span><span id="line-820"> private boolean zookeeperMode = false;</span> |
| <span class="source-line-no">821</span><span id="line-821"></span> |
| <span class="source-line-no">822</span><span id="line-822"> /**</span> |
| <span class="source-line-no">823</span><span id="line-823"> * This is a Map of table to timeout. The timeout is for reading all regions in the table; i.e. we</span> |
| <span class="source-line-no">824</span><span id="line-824"> * aggregate time to fetch each region and it needs to be less than this value else we log an</span> |
| <span class="source-line-no">825</span><span id="line-825"> * ERROR.</span> |
| <span class="source-line-no">826</span><span id="line-826"> */</span> |
| <span class="source-line-no">827</span><span id="line-827"> private HashMap<String, Long> configuredReadTableTimeouts = new HashMap<>();</span> |
| <span class="source-line-no">828</span><span id="line-828"></span> |
| <span class="source-line-no">829</span><span id="line-829"> public static final String HBASE_CANARY_REGIONSERVER_ALL_REGIONS =</span> |
| <span class="source-line-no">830</span><span id="line-830"> "hbase.canary.regionserver_all_regions";</span> |
| <span class="source-line-no">831</span><span id="line-831"></span> |
| <span class="source-line-no">832</span><span id="line-832"> public static final String HBASE_CANARY_REGION_WRITE_SNIFFING =</span> |
| <span class="source-line-no">833</span><span id="line-833"> "hbase.canary.region.write.sniffing";</span> |
| <span class="source-line-no">834</span><span id="line-834"> public static final String HBASE_CANARY_REGION_WRITE_TABLE_TIMEOUT =</span> |
| <span class="source-line-no">835</span><span id="line-835"> "hbase.canary.region.write.table.timeout";</span> |
| <span class="source-line-no">836</span><span id="line-836"> public static final String HBASE_CANARY_REGION_WRITE_TABLE_NAME =</span> |
| <span class="source-line-no">837</span><span id="line-837"> "hbase.canary.region.write.table.name";</span> |
| <span class="source-line-no">838</span><span id="line-838"> public static final String HBASE_CANARY_REGION_READ_TABLE_TIMEOUT =</span> |
| <span class="source-line-no">839</span><span id="line-839"> "hbase.canary.region.read.table.timeout";</span> |
| <span class="source-line-no">840</span><span id="line-840"></span> |
| <span class="source-line-no">841</span><span id="line-841"> public static final String HBASE_CANARY_ZOOKEEPER_PERMITTED_FAILURES =</span> |
| <span class="source-line-no">842</span><span id="line-842"> "hbase.canary.zookeeper.permitted.failures";</span> |
| <span class="source-line-no">843</span><span id="line-843"></span> |
| <span class="source-line-no">844</span><span id="line-844"> public static final String HBASE_CANARY_USE_REGEX = "hbase.canary.use.regex";</span> |
| <span class="source-line-no">845</span><span id="line-845"> public static final String HBASE_CANARY_TIMEOUT = "hbase.canary.timeout";</span> |
| <span class="source-line-no">846</span><span id="line-846"> public static final String HBASE_CANARY_FAIL_ON_ERROR = "hbase.canary.fail.on.error";</span> |
| <span class="source-line-no">847</span><span id="line-847"></span> |
| <span class="source-line-no">848</span><span id="line-848"> private ExecutorService executor; // threads to retrieve data from regionservers</span> |
| <span class="source-line-no">849</span><span id="line-849"></span> |
| <span class="source-line-no">850</span><span id="line-850"> public CanaryTool() {</span> |
| <span class="source-line-no">851</span><span id="line-851"> this(new ScheduledThreadPoolExecutor(1));</span> |
| <span class="source-line-no">852</span><span id="line-852"> }</span> |
| <span class="source-line-no">853</span><span id="line-853"></span> |
| <span class="source-line-no">854</span><span id="line-854"> public CanaryTool(ExecutorService executor) {</span> |
| <span class="source-line-no">855</span><span id="line-855"> this(executor, null);</span> |
| <span class="source-line-no">856</span><span id="line-856"> }</span> |
| <span class="source-line-no">857</span><span id="line-857"></span> |
| <span class="source-line-no">858</span><span id="line-858"> @InterfaceAudience.Private</span> |
| <span class="source-line-no">859</span><span id="line-859"> CanaryTool(ExecutorService executor, Sink sink) {</span> |
| <span class="source-line-no">860</span><span id="line-860"> this.executor = executor;</span> |
| <span class="source-line-no">861</span><span id="line-861"> this.sink = sink;</span> |
| <span class="source-line-no">862</span><span id="line-862"> }</span> |
| <span class="source-line-no">863</span><span id="line-863"></span> |
| <span class="source-line-no">864</span><span id="line-864"> CanaryTool(Configuration conf, ExecutorService executor) {</span> |
| <span class="source-line-no">865</span><span id="line-865"> this(conf, executor, null);</span> |
| <span class="source-line-no">866</span><span id="line-866"> }</span> |
| <span class="source-line-no">867</span><span id="line-867"></span> |
| <span class="source-line-no">868</span><span id="line-868"> CanaryTool(Configuration conf, ExecutorService executor, Sink sink) {</span> |
| <span class="source-line-no">869</span><span id="line-869"> this(executor, sink);</span> |
| <span class="source-line-no">870</span><span id="line-870"> setConf(conf);</span> |
| <span class="source-line-no">871</span><span id="line-871"> }</span> |
| <span class="source-line-no">872</span><span id="line-872"></span> |
| <span class="source-line-no">873</span><span id="line-873"> @Override</span> |
| <span class="source-line-no">874</span><span id="line-874"> public Configuration getConf() {</span> |
| <span class="source-line-no">875</span><span id="line-875"> return conf;</span> |
| <span class="source-line-no">876</span><span id="line-876"> }</span> |
| <span class="source-line-no">877</span><span id="line-877"></span> |
| <span class="source-line-no">878</span><span id="line-878"> @Override</span> |
| <span class="source-line-no">879</span><span id="line-879"> public void setConf(Configuration conf) {</span> |
| <span class="source-line-no">880</span><span id="line-880"> if (conf == null) {</span> |
| <span class="source-line-no">881</span><span id="line-881"> conf = HBaseConfiguration.create();</span> |
| <span class="source-line-no">882</span><span id="line-882"> }</span> |
| <span class="source-line-no">883</span><span id="line-883"> this.conf = conf;</span> |
| <span class="source-line-no">884</span><span id="line-884"> }</span> |
| <span class="source-line-no">885</span><span id="line-885"></span> |
| <span class="source-line-no">886</span><span id="line-886"> private int parseArgs(String[] args) {</span> |
| <span class="source-line-no">887</span><span id="line-887"> int index = -1;</span> |
| <span class="source-line-no">888</span><span id="line-888"> long permittedFailures = 0;</span> |
| <span class="source-line-no">889</span><span id="line-889"> boolean regionServerAllRegions = false, writeSniffing = false;</span> |
| <span class="source-line-no">890</span><span id="line-890"> String readTableTimeoutsStr = null;</span> |
| <span class="source-line-no">891</span><span id="line-891"> // Process command line args</span> |
| <span class="source-line-no">892</span><span id="line-892"> for (int i = 0; i < args.length; i++) {</span> |
| <span class="source-line-no">893</span><span id="line-893"> String cmd = args[i];</span> |
| <span class="source-line-no">894</span><span id="line-894"> if (cmd.startsWith("-")) {</span> |
| <span class="source-line-no">895</span><span id="line-895"> if (index >= 0) {</span> |
| <span class="source-line-no">896</span><span id="line-896"> // command line args must be in the form: [opts] [table 1 [table 2 ...]]</span> |
| <span class="source-line-no">897</span><span id="line-897"> System.err.println("Invalid command line options");</span> |
| <span class="source-line-no">898</span><span id="line-898"> printUsageAndExit();</span> |
| <span class="source-line-no">899</span><span id="line-899"> }</span> |
| <span class="source-line-no">900</span><span id="line-900"> if (cmd.equals("-help") || cmd.equals("-h")) {</span> |
| <span class="source-line-no">901</span><span id="line-901"> // user asked for help, print the help and quit.</span> |
| <span class="source-line-no">902</span><span id="line-902"> printUsageAndExit();</span> |
| <span class="source-line-no">903</span><span id="line-903"> } else if (cmd.equals("-daemon") && interval == 0) {</span> |
| <span class="source-line-no">904</span><span id="line-904"> // user asked for daemon mode, set a default interval between checks</span> |
| <span class="source-line-no">905</span><span id="line-905"> interval = DEFAULT_INTERVAL;</span> |
| <span class="source-line-no">906</span><span id="line-906"> } else if (cmd.equals("-interval")) {</span> |
| <span class="source-line-no">907</span><span id="line-907"> // user has specified an interval for canary breaths (-interval N)</span> |
| <span class="source-line-no">908</span><span id="line-908"> i++;</span> |
| <span class="source-line-no">909</span><span id="line-909"></span> |
| <span class="source-line-no">910</span><span id="line-910"> if (i == args.length) {</span> |
| <span class="source-line-no">911</span><span id="line-911"> System.err.println("-interval takes a numeric seconds value argument.");</span> |
| <span class="source-line-no">912</span><span id="line-912"> printUsageAndExit();</span> |
| <span class="source-line-no">913</span><span id="line-913"> }</span> |
| <span class="source-line-no">914</span><span id="line-914"> try {</span> |
| <span class="source-line-no">915</span><span id="line-915"> interval = Long.parseLong(args[i]) * 1000;</span> |
| <span class="source-line-no">916</span><span id="line-916"> } catch (NumberFormatException e) {</span> |
| <span class="source-line-no">917</span><span id="line-917"> System.err.println("-interval needs a numeric value argument.");</span> |
| <span class="source-line-no">918</span><span id="line-918"> printUsageAndExit();</span> |
| <span class="source-line-no">919</span><span id="line-919"> }</span> |
| <span class="source-line-no">920</span><span id="line-920"> } else if (cmd.equals("-zookeeper")) {</span> |
| <span class="source-line-no">921</span><span id="line-921"> this.zookeeperMode = true;</span> |
| <span class="source-line-no">922</span><span id="line-922"> } else if (cmd.equals("-regionserver")) {</span> |
| <span class="source-line-no">923</span><span id="line-923"> this.regionServerMode = true;</span> |
| <span class="source-line-no">924</span><span id="line-924"> } else if (cmd.equals("-allRegions")) {</span> |
| <span class="source-line-no">925</span><span id="line-925"> conf.setBoolean(HBASE_CANARY_REGIONSERVER_ALL_REGIONS, true);</span> |
| <span class="source-line-no">926</span><span id="line-926"> regionServerAllRegions = true;</span> |
| <span class="source-line-no">927</span><span id="line-927"> } else if (cmd.equals("-writeSniffing")) {</span> |
| <span class="source-line-no">928</span><span id="line-928"> writeSniffing = true;</span> |
| <span class="source-line-no">929</span><span id="line-929"> conf.setBoolean(HBASE_CANARY_REGION_WRITE_SNIFFING, true);</span> |
| <span class="source-line-no">930</span><span id="line-930"> } else if (cmd.equals("-treatFailureAsError") || cmd.equals("-failureAsError")) {</span> |
| <span class="source-line-no">931</span><span id="line-931"> conf.setBoolean(HBASE_CANARY_FAIL_ON_ERROR, true);</span> |
| <span class="source-line-no">932</span><span id="line-932"> } else if (cmd.equals("-e")) {</span> |
| <span class="source-line-no">933</span><span id="line-933"> conf.setBoolean(HBASE_CANARY_USE_REGEX, true);</span> |
| <span class="source-line-no">934</span><span id="line-934"> } else if (cmd.equals("-t")) {</span> |
| <span class="source-line-no">935</span><span id="line-935"> i++;</span> |
| <span class="source-line-no">936</span><span id="line-936"></span> |
| <span class="source-line-no">937</span><span id="line-937"> if (i == args.length) {</span> |
| <span class="source-line-no">938</span><span id="line-938"> System.err.println("-t takes a numeric milliseconds value argument.");</span> |
| <span class="source-line-no">939</span><span id="line-939"> printUsageAndExit();</span> |
| <span class="source-line-no">940</span><span id="line-940"> }</span> |
| <span class="source-line-no">941</span><span id="line-941"> long timeout = 0;</span> |
| <span class="source-line-no">942</span><span id="line-942"> try {</span> |
| <span class="source-line-no">943</span><span id="line-943"> timeout = Long.parseLong(args[i]);</span> |
| <span class="source-line-no">944</span><span id="line-944"> } catch (NumberFormatException e) {</span> |
| <span class="source-line-no">945</span><span id="line-945"> System.err.println("-t takes a numeric milliseconds value argument.");</span> |
| <span class="source-line-no">946</span><span id="line-946"> printUsageAndExit();</span> |
| <span class="source-line-no">947</span><span id="line-947"> }</span> |
| <span class="source-line-no">948</span><span id="line-948"> conf.setLong(HBASE_CANARY_TIMEOUT, timeout);</span> |
| <span class="source-line-no">949</span><span id="line-949"> } else if (cmd.equals("-writeTableTimeout")) {</span> |
| <span class="source-line-no">950</span><span id="line-950"> i++;</span> |
| <span class="source-line-no">951</span><span id="line-951"></span> |
| <span class="source-line-no">952</span><span id="line-952"> if (i == args.length) {</span> |
| <span class="source-line-no">953</span><span id="line-953"> System.err.println("-writeTableTimeout takes a numeric milliseconds value argument.");</span> |
| <span class="source-line-no">954</span><span id="line-954"> printUsageAndExit();</span> |
| <span class="source-line-no">955</span><span id="line-955"> }</span> |
| <span class="source-line-no">956</span><span id="line-956"> long configuredWriteTableTimeout = 0;</span> |
| <span class="source-line-no">957</span><span id="line-957"> try {</span> |
| <span class="source-line-no">958</span><span id="line-958"> configuredWriteTableTimeout = Long.parseLong(args[i]);</span> |
| <span class="source-line-no">959</span><span id="line-959"> } catch (NumberFormatException e) {</span> |
| <span class="source-line-no">960</span><span id="line-960"> System.err.println("-writeTableTimeout takes a numeric milliseconds value argument.");</span> |
| <span class="source-line-no">961</span><span id="line-961"> printUsageAndExit();</span> |
| <span class="source-line-no">962</span><span id="line-962"> }</span> |
| <span class="source-line-no">963</span><span id="line-963"> conf.setLong(HBASE_CANARY_REGION_WRITE_TABLE_TIMEOUT, configuredWriteTableTimeout);</span> |
| <span class="source-line-no">964</span><span id="line-964"> } else if (cmd.equals("-writeTable")) {</span> |
| <span class="source-line-no">965</span><span id="line-965"> i++;</span> |
| <span class="source-line-no">966</span><span id="line-966"></span> |
| <span class="source-line-no">967</span><span id="line-967"> if (i == args.length) {</span> |
| <span class="source-line-no">968</span><span id="line-968"> System.err.println("-writeTable takes a string tablename value argument.");</span> |
| <span class="source-line-no">969</span><span id="line-969"> printUsageAndExit();</span> |
| <span class="source-line-no">970</span><span id="line-970"> }</span> |
| <span class="source-line-no">971</span><span id="line-971"> conf.set(HBASE_CANARY_REGION_WRITE_TABLE_NAME, args[i]);</span> |
| <span class="source-line-no">972</span><span id="line-972"> } else if (cmd.equals("-f")) {</span> |
| <span class="source-line-no">973</span><span id="line-973"> i++;</span> |
| <span class="source-line-no">974</span><span id="line-974"> if (i == args.length) {</span> |
| <span class="source-line-no">975</span><span id="line-975"> System.err.println("-f needs a boolean value argument (true|false).");</span> |
| <span class="source-line-no">976</span><span id="line-976"> printUsageAndExit();</span> |
| <span class="source-line-no">977</span><span id="line-977"> }</span> |
| <span class="source-line-no">978</span><span id="line-978"></span> |
| <span class="source-line-no">979</span><span id="line-979"> conf.setBoolean(HBASE_CANARY_FAIL_ON_ERROR, Boolean.parseBoolean(args[i]));</span> |
| <span class="source-line-no">980</span><span id="line-980"> } else if (cmd.equals("-readTableTimeouts")) {</span> |
| <span class="source-line-no">981</span><span id="line-981"> i++;</span> |
| <span class="source-line-no">982</span><span id="line-982"> if (i == args.length) {</span> |
| <span class="source-line-no">983</span><span id="line-983"> System.err.println("-readTableTimeouts needs a comma-separated list of read "</span> |
| <span class="source-line-no">984</span><span id="line-984"> + "millisecond timeouts per table (without spaces).");</span> |
| <span class="source-line-no">985</span><span id="line-985"> printUsageAndExit();</span> |
| <span class="source-line-no">986</span><span id="line-986"> }</span> |
| <span class="source-line-no">987</span><span id="line-987"> readTableTimeoutsStr = args[i];</span> |
| <span class="source-line-no">988</span><span id="line-988"> conf.set(HBASE_CANARY_REGION_READ_TABLE_TIMEOUT, readTableTimeoutsStr);</span> |
| <span class="source-line-no">989</span><span id="line-989"> } else if (cmd.equals("-permittedZookeeperFailures")) {</span> |
| <span class="source-line-no">990</span><span id="line-990"> i++;</span> |
| <span class="source-line-no">991</span><span id="line-991"></span> |
| <span class="source-line-no">992</span><span id="line-992"> if (i == args.length) {</span> |
| <span class="source-line-no">993</span><span id="line-993"> System.err.println("-permittedZookeeperFailures needs a numeric value argument.");</span> |
| <span class="source-line-no">994</span><span id="line-994"> printUsageAndExit();</span> |
| <span class="source-line-no">995</span><span id="line-995"> }</span> |
| <span class="source-line-no">996</span><span id="line-996"> try {</span> |
| <span class="source-line-no">997</span><span id="line-997"> permittedFailures = Long.parseLong(args[i]);</span> |
| <span class="source-line-no">998</span><span id="line-998"> } catch (NumberFormatException e) {</span> |
| <span class="source-line-no">999</span><span id="line-999"> System.err.println("-permittedZookeeperFailures needs a numeric value argument.");</span> |
| <span class="source-line-no">1000</span><span id="line-1000"> printUsageAndExit();</span> |
| <span class="source-line-no">1001</span><span id="line-1001"> }</span> |
| <span class="source-line-no">1002</span><span id="line-1002"> conf.setLong(HBASE_CANARY_ZOOKEEPER_PERMITTED_FAILURES, permittedFailures);</span> |
| <span class="source-line-no">1003</span><span id="line-1003"> } else {</span> |
| <span class="source-line-no">1004</span><span id="line-1004"> // no options match</span> |
| <span class="source-line-no">1005</span><span id="line-1005"> System.err.println(cmd + " options is invalid.");</span> |
| <span class="source-line-no">1006</span><span id="line-1006"> printUsageAndExit();</span> |
| <span class="source-line-no">1007</span><span id="line-1007"> }</span> |
| <span class="source-line-no">1008</span><span id="line-1008"> } else if (index < 0) {</span> |
| <span class="source-line-no">1009</span><span id="line-1009"> // keep track of first table name specified by the user</span> |
| <span class="source-line-no">1010</span><span id="line-1010"> index = i;</span> |
| <span class="source-line-no">1011</span><span id="line-1011"> }</span> |
| <span class="source-line-no">1012</span><span id="line-1012"> }</span> |
| <span class="source-line-no">1013</span><span id="line-1013"> if (regionServerAllRegions && !this.regionServerMode) {</span> |
| <span class="source-line-no">1014</span><span id="line-1014"> System.err.println("-allRegions can only be specified in regionserver mode.");</span> |
| <span class="source-line-no">1015</span><span id="line-1015"> printUsageAndExit();</span> |
| <span class="source-line-no">1016</span><span id="line-1016"> }</span> |
| <span class="source-line-no">1017</span><span id="line-1017"> if (this.zookeeperMode) {</span> |
| <span class="source-line-no">1018</span><span id="line-1018"> if (this.regionServerMode || regionServerAllRegions || writeSniffing) {</span> |
| <span class="source-line-no">1019</span><span id="line-1019"> System.err.println("-zookeeper is exclusive and cannot be combined with " + "other modes.");</span> |
| <span class="source-line-no">1020</span><span id="line-1020"> printUsageAndExit();</span> |
| <span class="source-line-no">1021</span><span id="line-1021"> }</span> |
| <span class="source-line-no">1022</span><span id="line-1022"> }</span> |
| <span class="source-line-no">1023</span><span id="line-1023"> if (permittedFailures != 0 && !this.zookeeperMode) {</span> |
| <span class="source-line-no">1024</span><span id="line-1024"> System.err.println("-permittedZookeeperFailures requires -zookeeper mode.");</span> |
| <span class="source-line-no">1025</span><span id="line-1025"> printUsageAndExit();</span> |
| <span class="source-line-no">1026</span><span id="line-1026"> }</span> |
| <span class="source-line-no">1027</span><span id="line-1027"> if (readTableTimeoutsStr != null && (this.regionServerMode || this.zookeeperMode)) {</span> |
| <span class="source-line-no">1028</span><span id="line-1028"> System.err.println("-readTableTimeouts can only be configured in region mode.");</span> |
| <span class="source-line-no">1029</span><span id="line-1029"> printUsageAndExit();</span> |
| <span class="source-line-no">1030</span><span id="line-1030"> }</span> |
| <span class="source-line-no">1031</span><span id="line-1031"> return index;</span> |
| <span class="source-line-no">1032</span><span id="line-1032"> }</span> |
| <span class="source-line-no">1033</span><span id="line-1033"></span> |
| <span class="source-line-no">1034</span><span id="line-1034"> @Override</span> |
| <span class="source-line-no">1035</span><span id="line-1035"> public int run(String[] args) throws Exception {</span> |
| <span class="source-line-no">1036</span><span id="line-1036"> int index = parseArgs(args);</span> |
| <span class="source-line-no">1037</span><span id="line-1037"> String[] monitorTargets = null;</span> |
| <span class="source-line-no">1038</span><span id="line-1038"></span> |
| <span class="source-line-no">1039</span><span id="line-1039"> if (index >= 0) {</span> |
| <span class="source-line-no">1040</span><span id="line-1040"> int length = args.length - index;</span> |
| <span class="source-line-no">1041</span><span id="line-1041"> monitorTargets = new String[length];</span> |
| <span class="source-line-no">1042</span><span id="line-1042"> System.arraycopy(args, index, monitorTargets, 0, length);</span> |
| <span class="source-line-no">1043</span><span id="line-1043"> }</span> |
| <span class="source-line-no">1044</span><span id="line-1044"> if (interval > 0) {</span> |
| <span class="source-line-no">1045</span><span id="line-1045"> // Only show the web page in daemon mode</span> |
| <span class="source-line-no">1046</span><span id="line-1046"> putUpWebUI();</span> |
| <span class="source-line-no">1047</span><span id="line-1047"> }</span> |
| <span class="source-line-no">1048</span><span id="line-1048"> if (zookeeperMode) {</span> |
| <span class="source-line-no">1049</span><span id="line-1049"> return checkZooKeeper();</span> |
| <span class="source-line-no">1050</span><span id="line-1050"> } else if (regionServerMode) {</span> |
| <span class="source-line-no">1051</span><span id="line-1051"> return checkRegionServers(monitorTargets);</span> |
| <span class="source-line-no">1052</span><span id="line-1052"> } else {</span> |
| <span class="source-line-no">1053</span><span id="line-1053"> return checkRegions(monitorTargets);</span> |
| <span class="source-line-no">1054</span><span id="line-1054"> }</span> |
| <span class="source-line-no">1055</span><span id="line-1055"> }</span> |
| <span class="source-line-no">1056</span><span id="line-1056"></span> |
| <span class="source-line-no">1057</span><span id="line-1057"> private int runMonitor(String[] monitorTargets) throws Exception {</span> |
| <span class="source-line-no">1058</span><span id="line-1058"> ChoreService choreService = null;</span> |
| <span class="source-line-no">1059</span><span id="line-1059"></span> |
| <span class="source-line-no">1060</span><span id="line-1060"> // Launches chore for refreshing kerberos credentials if security is enabled.</span> |
| <span class="source-line-no">1061</span><span id="line-1061"> // Please see http://hbase.apache.org/book.html#_running_canary_in_a_kerberos_enabled_cluster</span> |
| <span class="source-line-no">1062</span><span id="line-1062"> // for more details.</span> |
| <span class="source-line-no">1063</span><span id="line-1063"> final ScheduledChore authChore = AuthUtil.getAuthChore(conf);</span> |
| <span class="source-line-no">1064</span><span id="line-1064"> if (authChore != null) {</span> |
| <span class="source-line-no">1065</span><span id="line-1065"> choreService = new ChoreService("CANARY_TOOL");</span> |
| <span class="source-line-no">1066</span><span id="line-1066"> choreService.scheduleChore(authChore);</span> |
| <span class="source-line-no">1067</span><span id="line-1067"> }</span> |
| <span class="source-line-no">1068</span><span id="line-1068"></span> |
| <span class="source-line-no">1069</span><span id="line-1069"> // Start to prepare the stuffs</span> |
| <span class="source-line-no">1070</span><span id="line-1070"> Monitor monitor = null;</span> |
| <span class="source-line-no">1071</span><span id="line-1071"> Thread monitorThread;</span> |
| <span class="source-line-no">1072</span><span id="line-1072"> long startTime = 0;</span> |
| <span class="source-line-no">1073</span><span id="line-1073"> long currentTimeLength = 0;</span> |
| <span class="source-line-no">1074</span><span id="line-1074"> boolean failOnError = conf.getBoolean(HBASE_CANARY_FAIL_ON_ERROR, true);</span> |
| <span class="source-line-no">1075</span><span id="line-1075"> long timeout = conf.getLong(HBASE_CANARY_TIMEOUT, DEFAULT_TIMEOUT);</span> |
| <span class="source-line-no">1076</span><span id="line-1076"> // Get a connection to use in below.</span> |
| <span class="source-line-no">1077</span><span id="line-1077"> try (Connection connection = ConnectionFactory.createConnection(this.conf)) {</span> |
| <span class="source-line-no">1078</span><span id="line-1078"> do {</span> |
| <span class="source-line-no">1079</span><span id="line-1079"> // Do monitor !!</span> |
| <span class="source-line-no">1080</span><span id="line-1080"> try {</span> |
| <span class="source-line-no">1081</span><span id="line-1081"> monitor = this.newMonitor(connection, monitorTargets);</span> |
| <span class="source-line-no">1082</span><span id="line-1082"> startTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1083</span><span id="line-1083"> monitorThread = new Thread(monitor, "CanaryMonitor-" + startTime);</span> |
| <span class="source-line-no">1084</span><span id="line-1084"> monitorThread.start();</span> |
| <span class="source-line-no">1085</span><span id="line-1085"> while (!monitor.isDone()) {</span> |
| <span class="source-line-no">1086</span><span id="line-1086"> // wait for 1 sec</span> |
| <span class="source-line-no">1087</span><span id="line-1087"> Thread.sleep(1000);</span> |
| <span class="source-line-no">1088</span><span id="line-1088"> // exit if any error occurs</span> |
| <span class="source-line-no">1089</span><span id="line-1089"> if (failOnError && monitor.hasError()) {</span> |
| <span class="source-line-no">1090</span><span id="line-1090"> monitorThread.interrupt();</span> |
| <span class="source-line-no">1091</span><span id="line-1091"> if (monitor.initialized) {</span> |
| <span class="source-line-no">1092</span><span id="line-1092"> return monitor.errorCode;</span> |
| <span class="source-line-no">1093</span><span id="line-1093"> } else {</span> |
| <span class="source-line-no">1094</span><span id="line-1094"> return INIT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1095</span><span id="line-1095"> }</span> |
| <span class="source-line-no">1096</span><span id="line-1096"> }</span> |
| <span class="source-line-no">1097</span><span id="line-1097"> currentTimeLength = EnvironmentEdgeManager.currentTime() - startTime;</span> |
| <span class="source-line-no">1098</span><span id="line-1098"> if (currentTimeLength > timeout) {</span> |
| <span class="source-line-no">1099</span><span id="line-1099"> LOG.error("The monitor is running too long (" + currentTimeLength</span> |
| <span class="source-line-no">1100</span><span id="line-1100"> + ") after timeout limit:" + timeout + " will be killed itself !!");</span> |
| <span class="source-line-no">1101</span><span id="line-1101"> monitorThread.interrupt();</span> |
| <span class="source-line-no">1102</span><span id="line-1102"> if (monitor.initialized) {</span> |
| <span class="source-line-no">1103</span><span id="line-1103"> return TIMEOUT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1104</span><span id="line-1104"> } else {</span> |
| <span class="source-line-no">1105</span><span id="line-1105"> return INIT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1106</span><span id="line-1106"> }</span> |
| <span class="source-line-no">1107</span><span id="line-1107"> }</span> |
| <span class="source-line-no">1108</span><span id="line-1108"> }</span> |
| <span class="source-line-no">1109</span><span id="line-1109"></span> |
| <span class="source-line-no">1110</span><span id="line-1110"> if (failOnError && monitor.finalCheckForErrors()) {</span> |
| <span class="source-line-no">1111</span><span id="line-1111"> monitorThread.interrupt();</span> |
| <span class="source-line-no">1112</span><span id="line-1112"> return monitor.errorCode;</span> |
| <span class="source-line-no">1113</span><span id="line-1113"> }</span> |
| <span class="source-line-no">1114</span><span id="line-1114"> } finally {</span> |
| <span class="source-line-no">1115</span><span id="line-1115"> if (monitor != null) {</span> |
| <span class="source-line-no">1116</span><span id="line-1116"> monitor.close();</span> |
| <span class="source-line-no">1117</span><span id="line-1117"> }</span> |
| <span class="source-line-no">1118</span><span id="line-1118"> }</span> |
| <span class="source-line-no">1119</span><span id="line-1119"></span> |
| <span class="source-line-no">1120</span><span id="line-1120"> Thread.sleep(interval);</span> |
| <span class="source-line-no">1121</span><span id="line-1121"> } while (interval > 0);</span> |
| <span class="source-line-no">1122</span><span id="line-1122"> } // try-with-resources close</span> |
| <span class="source-line-no">1123</span><span id="line-1123"></span> |
| <span class="source-line-no">1124</span><span id="line-1124"> if (choreService != null) {</span> |
| <span class="source-line-no">1125</span><span id="line-1125"> choreService.shutdown();</span> |
| <span class="source-line-no">1126</span><span id="line-1126"> }</span> |
| <span class="source-line-no">1127</span><span id="line-1127"> return monitor.errorCode;</span> |
| <span class="source-line-no">1128</span><span id="line-1128"> }</span> |
| <span class="source-line-no">1129</span><span id="line-1129"></span> |
| <span class="source-line-no">1130</span><span id="line-1130"> @Override</span> |
| <span class="source-line-no">1131</span><span id="line-1131"> public Map<String, String> getReadFailures() {</span> |
| <span class="source-line-no">1132</span><span id="line-1132"> return sink.getReadFailures();</span> |
| <span class="source-line-no">1133</span><span id="line-1133"> }</span> |
| <span class="source-line-no">1134</span><span id="line-1134"></span> |
| <span class="source-line-no">1135</span><span id="line-1135"> @Override</span> |
| <span class="source-line-no">1136</span><span id="line-1136"> public Map<String, String> getWriteFailures() {</span> |
| <span class="source-line-no">1137</span><span id="line-1137"> return sink.getWriteFailures();</span> |
| <span class="source-line-no">1138</span><span id="line-1138"> }</span> |
| <span class="source-line-no">1139</span><span id="line-1139"></span> |
| <span class="source-line-no">1140</span><span id="line-1140"> /**</span> |
| <span class="source-line-no">1141</span><span id="line-1141"> * Return a CanaryTool.Sink object containing the detailed results of the canary run. The Sink may</span> |
| <span class="source-line-no">1142</span><span id="line-1142"> * not have been created if a Monitor thread is not yet running.</span> |
| <span class="source-line-no">1143</span><span id="line-1143"> * @return the active Sink if one exists, null otherwise.</span> |
| <span class="source-line-no">1144</span><span id="line-1144"> */</span> |
| <span class="source-line-no">1145</span><span id="line-1145"> public Sink getActiveSink() {</span> |
| <span class="source-line-no">1146</span><span id="line-1146"> return sink;</span> |
| <span class="source-line-no">1147</span><span id="line-1147"> }</span> |
| <span class="source-line-no">1148</span><span id="line-1148"></span> |
| <span class="source-line-no">1149</span><span id="line-1149"> private void printUsageAndExit() {</span> |
| <span class="source-line-no">1150</span><span id="line-1150"> System.err.println(</span> |
| <span class="source-line-no">1151</span><span id="line-1151"> "Usage: canary [OPTIONS] [<TABLE1> [<TABLE2]...] | [<REGIONSERVER1> [<REGIONSERVER2]..]");</span> |
| <span class="source-line-no">1152</span><span id="line-1152"> System.err.println("Where [OPTIONS] are:");</span> |
| <span class="source-line-no">1153</span><span id="line-1153"> System.err.println(" -h,-help show this help and exit.");</span> |
| <span class="source-line-no">1154</span><span id="line-1154"> System.err.println(</span> |
| <span class="source-line-no">1155</span><span id="line-1155"> " -regionserver set 'regionserver mode'; gets row from random region on " + "server");</span> |
| <span class="source-line-no">1156</span><span id="line-1156"> System.err.println(</span> |
| <span class="source-line-no">1157</span><span id="line-1157"> " -allRegions get from ALL regions when 'regionserver mode', not just " + "random one.");</span> |
| <span class="source-line-no">1158</span><span id="line-1158"> System.err.println(" -zookeeper set 'zookeeper mode'; grab zookeeper.znode.parent on "</span> |
| <span class="source-line-no">1159</span><span id="line-1159"> + "each ensemble member");</span> |
| <span class="source-line-no">1160</span><span id="line-1160"> System.err.println(" -daemon continuous check at defined intervals.");</span> |
| <span class="source-line-no">1161</span><span id="line-1161"> System.err.println(" -interval <N> interval between checks in seconds");</span> |
| <span class="source-line-no">1162</span><span id="line-1162"> System.err</span> |
| <span class="source-line-no">1163</span><span id="line-1163"> .println(" -e consider table/regionserver argument as regular " + "expression");</span> |
| <span class="source-line-no">1164</span><span id="line-1164"> System.err.println(" -f <B> exit on first error; default=true");</span> |
| <span class="source-line-no">1165</span><span id="line-1165"> System.err.println(" -failureAsError treat read/write failure as error");</span> |
| <span class="source-line-no">1166</span><span id="line-1166"> System.err.println(" -t <N> timeout for canary-test run; default=600000ms");</span> |
| <span class="source-line-no">1167</span><span id="line-1167"> System.err.println(" -writeSniffing enable write sniffing");</span> |
| <span class="source-line-no">1168</span><span id="line-1168"> System.err.println(" -writeTable the table used for write sniffing; default=hbase:canary");</span> |
| <span class="source-line-no">1169</span><span id="line-1169"> System.err.println(" -writeTableTimeout <N> timeout for writeTable; default=600000ms");</span> |
| <span class="source-line-no">1170</span><span id="line-1170"> System.err.println(</span> |
| <span class="source-line-no">1171</span><span id="line-1171"> " -readTableTimeouts <tableName>=<read timeout>," + "<tableName>=<read timeout>,...");</span> |
| <span class="source-line-no">1172</span><span id="line-1172"> System.err</span> |
| <span class="source-line-no">1173</span><span id="line-1173"> .println(" comma-separated list of table read timeouts " + "(no spaces);");</span> |
| <span class="source-line-no">1174</span><span id="line-1174"> System.err.println(" logs 'ERROR' if takes longer. default=600000ms");</span> |
| <span class="source-line-no">1175</span><span id="line-1175"> System.err.println(" -permittedZookeeperFailures <N> Ignore first N failures attempting to ");</span> |
| <span class="source-line-no">1176</span><span id="line-1176"> System.err.println(" connect to individual zookeeper nodes in ensemble");</span> |
| <span class="source-line-no">1177</span><span id="line-1177"> System.err.println("");</span> |
| <span class="source-line-no">1178</span><span id="line-1178"> System.err.println(" -D<configProperty>=<value> to assign or override configuration params");</span> |
| <span class="source-line-no">1179</span><span id="line-1179"> System.err.println(" -Dhbase.canary.read.raw.enabled=<true/false> Set to enable/disable "</span> |
| <span class="source-line-no">1180</span><span id="line-1180"> + "raw scan; default=false");</span> |
| <span class="source-line-no">1181</span><span id="line-1181"> System.err.println(</span> |
| <span class="source-line-no">1182</span><span id="line-1182"> " -Dhbase.canary.info.port=PORT_NUMBER Set for a Canary UI; " + "default=-1 (None)");</span> |
| <span class="source-line-no">1183</span><span id="line-1183"> System.err.println("");</span> |
| <span class="source-line-no">1184</span><span id="line-1184"> System.err.println(</span> |
| <span class="source-line-no">1185</span><span id="line-1185"> "Canary runs in one of three modes: region (default), regionserver, or " + "zookeeper.");</span> |
| <span class="source-line-no">1186</span><span id="line-1186"> System.err.println("To sniff/probe all regions, pass no arguments.");</span> |
| <span class="source-line-no">1187</span><span id="line-1187"> System.err.println("To sniff/probe all regions of a table, pass tablename.");</span> |
| <span class="source-line-no">1188</span><span id="line-1188"> System.err.println("To sniff/probe regionservers, pass -regionserver, etc.");</span> |
| <span class="source-line-no">1189</span><span id="line-1189"> System.err.println("See http://hbase.apache.org/book.html#_canary for Canary documentation.");</span> |
| <span class="source-line-no">1190</span><span id="line-1190"> System.exit(USAGE_EXIT_CODE);</span> |
| <span class="source-line-no">1191</span><span id="line-1191"> }</span> |
| <span class="source-line-no">1192</span><span id="line-1192"></span> |
| <span class="source-line-no">1193</span><span id="line-1193"> Sink getSink(Configuration configuration, Class clazz) {</span> |
| <span class="source-line-no">1194</span><span id="line-1194"> // In test context, this.sink might be set. Use it if non-null. For testing.</span> |
| <span class="source-line-no">1195</span><span id="line-1195"> if (this.sink == null) {</span> |
| <span class="source-line-no">1196</span><span id="line-1196"> this.sink = (Sink) ReflectionUtils</span> |
| <span class="source-line-no">1197</span><span id="line-1197"> .newInstance(configuration.getClass("hbase.canary.sink.class", clazz, Sink.class));</span> |
| <span class="source-line-no">1198</span><span id="line-1198"> }</span> |
| <span class="source-line-no">1199</span><span id="line-1199"> return this.sink;</span> |
| <span class="source-line-no">1200</span><span id="line-1200"> }</span> |
| <span class="source-line-no">1201</span><span id="line-1201"></span> |
| <span class="source-line-no">1202</span><span id="line-1202"> /**</span> |
| <span class="source-line-no">1203</span><span id="line-1203"> * Canary region mode-specific data structure which stores information about each region to be</span> |
| <span class="source-line-no">1204</span><span id="line-1204"> * scanned</span> |
| <span class="source-line-no">1205</span><span id="line-1205"> */</span> |
| <span class="source-line-no">1206</span><span id="line-1206"> public static class RegionTaskResult {</span> |
| <span class="source-line-no">1207</span><span id="line-1207"> private RegionInfo region;</span> |
| <span class="source-line-no">1208</span><span id="line-1208"> private TableName tableName;</span> |
| <span class="source-line-no">1209</span><span id="line-1209"> private ServerName serverName;</span> |
| <span class="source-line-no">1210</span><span id="line-1210"> private ColumnFamilyDescriptor column;</span> |
| <span class="source-line-no">1211</span><span id="line-1211"> private AtomicLong readLatency = null;</span> |
| <span class="source-line-no">1212</span><span id="line-1212"> private AtomicLong writeLatency = null;</span> |
| <span class="source-line-no">1213</span><span id="line-1213"> private boolean readSuccess = false;</span> |
| <span class="source-line-no">1214</span><span id="line-1214"> private boolean writeSuccess = false;</span> |
| <span class="source-line-no">1215</span><span id="line-1215"></span> |
| <span class="source-line-no">1216</span><span id="line-1216"> public RegionTaskResult(RegionInfo region, TableName tableName, ServerName serverName,</span> |
| <span class="source-line-no">1217</span><span id="line-1217"> ColumnFamilyDescriptor column) {</span> |
| <span class="source-line-no">1218</span><span id="line-1218"> this.region = region;</span> |
| <span class="source-line-no">1219</span><span id="line-1219"> this.tableName = tableName;</span> |
| <span class="source-line-no">1220</span><span id="line-1220"> this.serverName = serverName;</span> |
| <span class="source-line-no">1221</span><span id="line-1221"> this.column = column;</span> |
| <span class="source-line-no">1222</span><span id="line-1222"> }</span> |
| <span class="source-line-no">1223</span><span id="line-1223"></span> |
| <span class="source-line-no">1224</span><span id="line-1224"> public RegionInfo getRegionInfo() {</span> |
| <span class="source-line-no">1225</span><span id="line-1225"> return this.region;</span> |
| <span class="source-line-no">1226</span><span id="line-1226"> }</span> |
| <span class="source-line-no">1227</span><span id="line-1227"></span> |
| <span class="source-line-no">1228</span><span id="line-1228"> public String getRegionNameAsString() {</span> |
| <span class="source-line-no">1229</span><span id="line-1229"> return this.region.getRegionNameAsString();</span> |
| <span class="source-line-no">1230</span><span id="line-1230"> }</span> |
| <span class="source-line-no">1231</span><span id="line-1231"></span> |
| <span class="source-line-no">1232</span><span id="line-1232"> public TableName getTableName() {</span> |
| <span class="source-line-no">1233</span><span id="line-1233"> return this.tableName;</span> |
| <span class="source-line-no">1234</span><span id="line-1234"> }</span> |
| <span class="source-line-no">1235</span><span id="line-1235"></span> |
| <span class="source-line-no">1236</span><span id="line-1236"> public String getTableNameAsString() {</span> |
| <span class="source-line-no">1237</span><span id="line-1237"> return this.tableName.getNameAsString();</span> |
| <span class="source-line-no">1238</span><span id="line-1238"> }</span> |
| <span class="source-line-no">1239</span><span id="line-1239"></span> |
| <span class="source-line-no">1240</span><span id="line-1240"> public ServerName getServerName() {</span> |
| <span class="source-line-no">1241</span><span id="line-1241"> return this.serverName;</span> |
| <span class="source-line-no">1242</span><span id="line-1242"> }</span> |
| <span class="source-line-no">1243</span><span id="line-1243"></span> |
| <span class="source-line-no">1244</span><span id="line-1244"> public String getServerNameAsString() {</span> |
| <span class="source-line-no">1245</span><span id="line-1245"> return this.serverName.getServerName();</span> |
| <span class="source-line-no">1246</span><span id="line-1246"> }</span> |
| <span class="source-line-no">1247</span><span id="line-1247"></span> |
| <span class="source-line-no">1248</span><span id="line-1248"> public ColumnFamilyDescriptor getColumnFamily() {</span> |
| <span class="source-line-no">1249</span><span id="line-1249"> return this.column;</span> |
| <span class="source-line-no">1250</span><span id="line-1250"> }</span> |
| <span class="source-line-no">1251</span><span id="line-1251"></span> |
| <span class="source-line-no">1252</span><span id="line-1252"> public String getColumnFamilyNameAsString() {</span> |
| <span class="source-line-no">1253</span><span id="line-1253"> return this.column.getNameAsString();</span> |
| <span class="source-line-no">1254</span><span id="line-1254"> }</span> |
| <span class="source-line-no">1255</span><span id="line-1255"></span> |
| <span class="source-line-no">1256</span><span id="line-1256"> public long getReadLatency() {</span> |
| <span class="source-line-no">1257</span><span id="line-1257"> if (this.readLatency == null) {</span> |
| <span class="source-line-no">1258</span><span id="line-1258"> return -1;</span> |
| <span class="source-line-no">1259</span><span id="line-1259"> }</span> |
| <span class="source-line-no">1260</span><span id="line-1260"> return this.readLatency.get();</span> |
| <span class="source-line-no">1261</span><span id="line-1261"> }</span> |
| <span class="source-line-no">1262</span><span id="line-1262"></span> |
| <span class="source-line-no">1263</span><span id="line-1263"> public void setReadLatency(long readLatency) {</span> |
| <span class="source-line-no">1264</span><span id="line-1264"> if (this.readLatency != null) {</span> |
| <span class="source-line-no">1265</span><span id="line-1265"> this.readLatency.set(readLatency);</span> |
| <span class="source-line-no">1266</span><span id="line-1266"> } else {</span> |
| <span class="source-line-no">1267</span><span id="line-1267"> this.readLatency = new AtomicLong(readLatency);</span> |
| <span class="source-line-no">1268</span><span id="line-1268"> }</span> |
| <span class="source-line-no">1269</span><span id="line-1269"> }</span> |
| <span class="source-line-no">1270</span><span id="line-1270"></span> |
| <span class="source-line-no">1271</span><span id="line-1271"> public long getWriteLatency() {</span> |
| <span class="source-line-no">1272</span><span id="line-1272"> if (this.writeLatency == null) {</span> |
| <span class="source-line-no">1273</span><span id="line-1273"> return -1;</span> |
| <span class="source-line-no">1274</span><span id="line-1274"> }</span> |
| <span class="source-line-no">1275</span><span id="line-1275"> return this.writeLatency.get();</span> |
| <span class="source-line-no">1276</span><span id="line-1276"> }</span> |
| <span class="source-line-no">1277</span><span id="line-1277"></span> |
| <span class="source-line-no">1278</span><span id="line-1278"> public void setWriteLatency(long writeLatency) {</span> |
| <span class="source-line-no">1279</span><span id="line-1279"> if (this.writeLatency != null) {</span> |
| <span class="source-line-no">1280</span><span id="line-1280"> this.writeLatency.set(writeLatency);</span> |
| <span class="source-line-no">1281</span><span id="line-1281"> } else {</span> |
| <span class="source-line-no">1282</span><span id="line-1282"> this.writeLatency = new AtomicLong(writeLatency);</span> |
| <span class="source-line-no">1283</span><span id="line-1283"> }</span> |
| <span class="source-line-no">1284</span><span id="line-1284"> }</span> |
| <span class="source-line-no">1285</span><span id="line-1285"></span> |
| <span class="source-line-no">1286</span><span id="line-1286"> public boolean isReadSuccess() {</span> |
| <span class="source-line-no">1287</span><span id="line-1287"> return this.readSuccess;</span> |
| <span class="source-line-no">1288</span><span id="line-1288"> }</span> |
| <span class="source-line-no">1289</span><span id="line-1289"></span> |
| <span class="source-line-no">1290</span><span id="line-1290"> public void setReadSuccess() {</span> |
| <span class="source-line-no">1291</span><span id="line-1291"> this.readSuccess = true;</span> |
| <span class="source-line-no">1292</span><span id="line-1292"> }</span> |
| <span class="source-line-no">1293</span><span id="line-1293"></span> |
| <span class="source-line-no">1294</span><span id="line-1294"> public boolean isWriteSuccess() {</span> |
| <span class="source-line-no">1295</span><span id="line-1295"> return this.writeSuccess;</span> |
| <span class="source-line-no">1296</span><span id="line-1296"> }</span> |
| <span class="source-line-no">1297</span><span id="line-1297"></span> |
| <span class="source-line-no">1298</span><span id="line-1298"> public void setWriteSuccess() {</span> |
| <span class="source-line-no">1299</span><span id="line-1299"> this.writeSuccess = true;</span> |
| <span class="source-line-no">1300</span><span id="line-1300"> }</span> |
| <span class="source-line-no">1301</span><span id="line-1301"> }</span> |
| <span class="source-line-no">1302</span><span id="line-1302"></span> |
| <span class="source-line-no">1303</span><span id="line-1303"> /**</span> |
| <span class="source-line-no">1304</span><span id="line-1304"> * A Factory method for {@link Monitor}. Makes a RegionServerMonitor, or a ZooKeeperMonitor, or a</span> |
| <span class="source-line-no">1305</span><span id="line-1305"> * RegionMonitor.</span> |
| <span class="source-line-no">1306</span><span id="line-1306"> * @return a Monitor instance</span> |
| <span class="source-line-no">1307</span><span id="line-1307"> */</span> |
| <span class="source-line-no">1308</span><span id="line-1308"> private Monitor newMonitor(final Connection connection, String[] monitorTargets) {</span> |
| <span class="source-line-no">1309</span><span id="line-1309"> Monitor monitor;</span> |
| <span class="source-line-no">1310</span><span id="line-1310"> boolean useRegExp = conf.getBoolean(HBASE_CANARY_USE_REGEX, false);</span> |
| <span class="source-line-no">1311</span><span id="line-1311"> boolean regionServerAllRegions = conf.getBoolean(HBASE_CANARY_REGIONSERVER_ALL_REGIONS, false);</span> |
| <span class="source-line-no">1312</span><span id="line-1312"> boolean failOnError = conf.getBoolean(HBASE_CANARY_FAIL_ON_ERROR, true);</span> |
| <span class="source-line-no">1313</span><span id="line-1313"> int permittedFailures = conf.getInt(HBASE_CANARY_ZOOKEEPER_PERMITTED_FAILURES, 0);</span> |
| <span class="source-line-no">1314</span><span id="line-1314"> boolean writeSniffing = conf.getBoolean(HBASE_CANARY_REGION_WRITE_SNIFFING, false);</span> |
| <span class="source-line-no">1315</span><span id="line-1315"> String writeTableName =</span> |
| <span class="source-line-no">1316</span><span id="line-1316"> conf.get(HBASE_CANARY_REGION_WRITE_TABLE_NAME, DEFAULT_WRITE_TABLE_NAME.getNameAsString());</span> |
| <span class="source-line-no">1317</span><span id="line-1317"> long configuredWriteTableTimeout =</span> |
| <span class="source-line-no">1318</span><span id="line-1318"> conf.getLong(HBASE_CANARY_REGION_WRITE_TABLE_TIMEOUT, DEFAULT_TIMEOUT);</span> |
| <span class="source-line-no">1319</span><span id="line-1319"></span> |
| <span class="source-line-no">1320</span><span id="line-1320"> if (this.regionServerMode) {</span> |
| <span class="source-line-no">1321</span><span id="line-1321"> monitor = new RegionServerMonitor(connection, monitorTargets, useRegExp,</span> |
| <span class="source-line-no">1322</span><span id="line-1322"> getSink(connection.getConfiguration(), RegionServerStdOutSink.class), this.executor,</span> |
| <span class="source-line-no">1323</span><span id="line-1323"> regionServerAllRegions, failOnError, permittedFailures);</span> |
| <span class="source-line-no">1324</span><span id="line-1324"></span> |
| <span class="source-line-no">1325</span><span id="line-1325"> } else if (this.zookeeperMode) {</span> |
| <span class="source-line-no">1326</span><span id="line-1326"> monitor = new ZookeeperMonitor(connection, monitorTargets, useRegExp,</span> |
| <span class="source-line-no">1327</span><span id="line-1327"> getSink(connection.getConfiguration(), ZookeeperStdOutSink.class), this.executor,</span> |
| <span class="source-line-no">1328</span><span id="line-1328"> failOnError, permittedFailures);</span> |
| <span class="source-line-no">1329</span><span id="line-1329"> } else {</span> |
| <span class="source-line-no">1330</span><span id="line-1330"> monitor = new RegionMonitor(connection, monitorTargets, useRegExp,</span> |
| <span class="source-line-no">1331</span><span id="line-1331"> getSink(connection.getConfiguration(), RegionStdOutSink.class), this.executor,</span> |
| <span class="source-line-no">1332</span><span id="line-1332"> writeSniffing, TableName.valueOf(writeTableName), failOnError, configuredReadTableTimeouts,</span> |
| <span class="source-line-no">1333</span><span id="line-1333"> configuredWriteTableTimeout, permittedFailures);</span> |
| <span class="source-line-no">1334</span><span id="line-1334"> }</span> |
| <span class="source-line-no">1335</span><span id="line-1335"> return monitor;</span> |
| <span class="source-line-no">1336</span><span id="line-1336"> }</span> |
| <span class="source-line-no">1337</span><span id="line-1337"></span> |
| <span class="source-line-no">1338</span><span id="line-1338"> private void populateReadTableTimeoutsMap(String configuredReadTableTimeoutsStr) {</span> |
| <span class="source-line-no">1339</span><span id="line-1339"> String[] tableTimeouts = configuredReadTableTimeoutsStr.split(",");</span> |
| <span class="source-line-no">1340</span><span id="line-1340"> for (String tT : tableTimeouts) {</span> |
| <span class="source-line-no">1341</span><span id="line-1341"> String[] nameTimeout = tT.split("=");</span> |
| <span class="source-line-no">1342</span><span id="line-1342"> if (nameTimeout.length < 2) {</span> |
| <span class="source-line-no">1343</span><span id="line-1343"> throw new IllegalArgumentException("Each -readTableTimeouts argument must be of the form "</span> |
| <span class="source-line-no">1344</span><span id="line-1344"> + "<tableName>=<read timeout> (without spaces).");</span> |
| <span class="source-line-no">1345</span><span id="line-1345"> }</span> |
| <span class="source-line-no">1346</span><span id="line-1346"> long timeoutVal;</span> |
| <span class="source-line-no">1347</span><span id="line-1347"> try {</span> |
| <span class="source-line-no">1348</span><span id="line-1348"> timeoutVal = Long.parseLong(nameTimeout[1]);</span> |
| <span class="source-line-no">1349</span><span id="line-1349"> } catch (NumberFormatException e) {</span> |
| <span class="source-line-no">1350</span><span id="line-1350"> throw new IllegalArgumentException(</span> |
| <span class="source-line-no">1351</span><span id="line-1351"> "-readTableTimeouts read timeout for each table" + " must be a numeric value argument.");</span> |
| <span class="source-line-no">1352</span><span id="line-1352"> }</span> |
| <span class="source-line-no">1353</span><span id="line-1353"> configuredReadTableTimeouts.put(nameTimeout[0], timeoutVal);</span> |
| <span class="source-line-no">1354</span><span id="line-1354"> }</span> |
| <span class="source-line-no">1355</span><span id="line-1355"> }</span> |
| <span class="source-line-no">1356</span><span id="line-1356"></span> |
| <span class="source-line-no">1357</span><span id="line-1357"> /**</span> |
| <span class="source-line-no">1358</span><span id="line-1358"> * A Monitor super-class can be extended by users</span> |
| <span class="source-line-no">1359</span><span id="line-1359"> */</span> |
| <span class="source-line-no">1360</span><span id="line-1360"> public static abstract class Monitor implements Runnable, Closeable {</span> |
| <span class="source-line-no">1361</span><span id="line-1361"> protected Connection connection;</span> |
| <span class="source-line-no">1362</span><span id="line-1362"> protected Admin admin;</span> |
| <span class="source-line-no">1363</span><span id="line-1363"> /**</span> |
| <span class="source-line-no">1364</span><span id="line-1364"> * 'Target' dependent on 'mode'. Could be Tables or RegionServers or ZNodes. Passed on the</span> |
| <span class="source-line-no">1365</span><span id="line-1365"> * command-line as arguments.</span> |
| <span class="source-line-no">1366</span><span id="line-1366"> */</span> |
| <span class="source-line-no">1367</span><span id="line-1367"> protected String[] targets;</span> |
| <span class="source-line-no">1368</span><span id="line-1368"> protected boolean useRegExp;</span> |
| <span class="source-line-no">1369</span><span id="line-1369"> protected boolean treatFailureAsError;</span> |
| <span class="source-line-no">1370</span><span id="line-1370"> protected boolean initialized = false;</span> |
| <span class="source-line-no">1371</span><span id="line-1371"></span> |
| <span class="source-line-no">1372</span><span id="line-1372"> protected boolean done = false;</span> |
| <span class="source-line-no">1373</span><span id="line-1373"> protected int errorCode = 0;</span> |
| <span class="source-line-no">1374</span><span id="line-1374"> protected long allowedFailures = 0;</span> |
| <span class="source-line-no">1375</span><span id="line-1375"> protected Sink sink;</span> |
| <span class="source-line-no">1376</span><span id="line-1376"> protected ExecutorService executor;</span> |
| <span class="source-line-no">1377</span><span id="line-1377"></span> |
| <span class="source-line-no">1378</span><span id="line-1378"> public boolean isDone() {</span> |
| <span class="source-line-no">1379</span><span id="line-1379"> return done;</span> |
| <span class="source-line-no">1380</span><span id="line-1380"> }</span> |
| <span class="source-line-no">1381</span><span id="line-1381"></span> |
| <span class="source-line-no">1382</span><span id="line-1382"> public boolean hasError() {</span> |
| <span class="source-line-no">1383</span><span id="line-1383"> return errorCode != 0;</span> |
| <span class="source-line-no">1384</span><span id="line-1384"> }</span> |
| <span class="source-line-no">1385</span><span id="line-1385"></span> |
| <span class="source-line-no">1386</span><span id="line-1386"> public boolean finalCheckForErrors() {</span> |
| <span class="source-line-no">1387</span><span id="line-1387"> if (errorCode != 0) {</span> |
| <span class="source-line-no">1388</span><span id="line-1388"> return true;</span> |
| <span class="source-line-no">1389</span><span id="line-1389"> }</span> |
| <span class="source-line-no">1390</span><span id="line-1390"> if (</span> |
| <span class="source-line-no">1391</span><span id="line-1391"> treatFailureAsError && (sink.getReadFailureCount() > allowedFailures</span> |
| <span class="source-line-no">1392</span><span id="line-1392"> || sink.getWriteFailureCount() > allowedFailures)</span> |
| <span class="source-line-no">1393</span><span id="line-1393"> ) {</span> |
| <span class="source-line-no">1394</span><span id="line-1394"> LOG.error("Too many failures detected, treating failure as error, failing the Canary.");</span> |
| <span class="source-line-no">1395</span><span id="line-1395"> errorCode = FAILURE_EXIT_CODE;</span> |
| <span class="source-line-no">1396</span><span id="line-1396"> return true;</span> |
| <span class="source-line-no">1397</span><span id="line-1397"> }</span> |
| <span class="source-line-no">1398</span><span id="line-1398"> return false;</span> |
| <span class="source-line-no">1399</span><span id="line-1399"> }</span> |
| <span class="source-line-no">1400</span><span id="line-1400"></span> |
| <span class="source-line-no">1401</span><span id="line-1401"> @Override</span> |
| <span class="source-line-no">1402</span><span id="line-1402"> public void close() throws IOException {</span> |
| <span class="source-line-no">1403</span><span id="line-1403"> this.sink.stop();</span> |
| <span class="source-line-no">1404</span><span id="line-1404"> if (this.admin != null) {</span> |
| <span class="source-line-no">1405</span><span id="line-1405"> this.admin.close();</span> |
| <span class="source-line-no">1406</span><span id="line-1406"> }</span> |
| <span class="source-line-no">1407</span><span id="line-1407"> }</span> |
| <span class="source-line-no">1408</span><span id="line-1408"></span> |
| <span class="source-line-no">1409</span><span id="line-1409"> protected Monitor(Connection connection, String[] monitorTargets, boolean useRegExp, Sink sink,</span> |
| <span class="source-line-no">1410</span><span id="line-1410"> ExecutorService executor, boolean treatFailureAsError, long allowedFailures) {</span> |
| <span class="source-line-no">1411</span><span id="line-1411"> if (null == connection) {</span> |
| <span class="source-line-no">1412</span><span id="line-1412"> throw new IllegalArgumentException("connection shall not be null");</span> |
| <span class="source-line-no">1413</span><span id="line-1413"> }</span> |
| <span class="source-line-no">1414</span><span id="line-1414"></span> |
| <span class="source-line-no">1415</span><span id="line-1415"> this.connection = connection;</span> |
| <span class="source-line-no">1416</span><span id="line-1416"> this.targets = monitorTargets;</span> |
| <span class="source-line-no">1417</span><span id="line-1417"> this.useRegExp = useRegExp;</span> |
| <span class="source-line-no">1418</span><span id="line-1418"> this.treatFailureAsError = treatFailureAsError;</span> |
| <span class="source-line-no">1419</span><span id="line-1419"> this.sink = sink;</span> |
| <span class="source-line-no">1420</span><span id="line-1420"> this.executor = executor;</span> |
| <span class="source-line-no">1421</span><span id="line-1421"> this.allowedFailures = allowedFailures;</span> |
| <span class="source-line-no">1422</span><span id="line-1422"> }</span> |
| <span class="source-line-no">1423</span><span id="line-1423"></span> |
| <span class="source-line-no">1424</span><span id="line-1424"> @Override</span> |
| <span class="source-line-no">1425</span><span id="line-1425"> public abstract void run();</span> |
| <span class="source-line-no">1426</span><span id="line-1426"></span> |
| <span class="source-line-no">1427</span><span id="line-1427"> protected boolean initAdmin() {</span> |
| <span class="source-line-no">1428</span><span id="line-1428"> if (null == this.admin) {</span> |
| <span class="source-line-no">1429</span><span id="line-1429"> try {</span> |
| <span class="source-line-no">1430</span><span id="line-1430"> this.admin = this.connection.getAdmin();</span> |
| <span class="source-line-no">1431</span><span id="line-1431"> } catch (Exception e) {</span> |
| <span class="source-line-no">1432</span><span id="line-1432"> LOG.error("Initial HBaseAdmin failed...", e);</span> |
| <span class="source-line-no">1433</span><span id="line-1433"> this.errorCode = INIT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1434</span><span id="line-1434"> }</span> |
| <span class="source-line-no">1435</span><span id="line-1435"> } else if (admin.isAborted()) {</span> |
| <span class="source-line-no">1436</span><span id="line-1436"> LOG.error("HBaseAdmin aborted");</span> |
| <span class="source-line-no">1437</span><span id="line-1437"> this.errorCode = INIT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1438</span><span id="line-1438"> }</span> |
| <span class="source-line-no">1439</span><span id="line-1439"> return !this.hasError();</span> |
| <span class="source-line-no">1440</span><span id="line-1440"> }</span> |
| <span class="source-line-no">1441</span><span id="line-1441"> }</span> |
| <span class="source-line-no">1442</span><span id="line-1442"></span> |
| <span class="source-line-no">1443</span><span id="line-1443"> /**</span> |
| <span class="source-line-no">1444</span><span id="line-1444"> * A monitor for region mode.</span> |
| <span class="source-line-no">1445</span><span id="line-1445"> */</span> |
| <span class="source-line-no">1446</span><span id="line-1446"> private static class RegionMonitor extends Monitor {</span> |
| <span class="source-line-no">1447</span><span id="line-1447"> // 10 minutes</span> |
| <span class="source-line-no">1448</span><span id="line-1448"> private static final int DEFAULT_WRITE_TABLE_CHECK_PERIOD = 10 * 60 * 1000;</span> |
| <span class="source-line-no">1449</span><span id="line-1449"> // 1 days</span> |
| <span class="source-line-no">1450</span><span id="line-1450"> private static final int DEFAULT_WRITE_DATA_TTL = 24 * 60 * 60;</span> |
| <span class="source-line-no">1451</span><span id="line-1451"></span> |
| <span class="source-line-no">1452</span><span id="line-1452"> private long lastCheckTime = -1;</span> |
| <span class="source-line-no">1453</span><span id="line-1453"> private boolean writeSniffing;</span> |
| <span class="source-line-no">1454</span><span id="line-1454"> private TableName writeTableName;</span> |
| <span class="source-line-no">1455</span><span id="line-1455"> private int writeDataTTL;</span> |
| <span class="source-line-no">1456</span><span id="line-1456"> private float regionsLowerLimit;</span> |
| <span class="source-line-no">1457</span><span id="line-1457"> private float regionsUpperLimit;</span> |
| <span class="source-line-no">1458</span><span id="line-1458"> private int checkPeriod;</span> |
| <span class="source-line-no">1459</span><span id="line-1459"> private boolean rawScanEnabled;</span> |
| <span class="source-line-no">1460</span><span id="line-1460"> private boolean readAllCF;</span> |
| <span class="source-line-no">1461</span><span id="line-1461"></span> |
| <span class="source-line-no">1462</span><span id="line-1462"> /**</span> |
| <span class="source-line-no">1463</span><span id="line-1463"> * This is a timeout per table. If read of each region in the table aggregated takes longer than</span> |
| <span class="source-line-no">1464</span><span id="line-1464"> * what is configured here, we log an ERROR rather than just an INFO.</span> |
| <span class="source-line-no">1465</span><span id="line-1465"> */</span> |
| <span class="source-line-no">1466</span><span id="line-1466"> private HashMap<String, Long> configuredReadTableTimeouts;</span> |
| <span class="source-line-no">1467</span><span id="line-1467"></span> |
| <span class="source-line-no">1468</span><span id="line-1468"> private long configuredWriteTableTimeout;</span> |
| <span class="source-line-no">1469</span><span id="line-1469"></span> |
| <span class="source-line-no">1470</span><span id="line-1470"> public RegionMonitor(Connection connection, String[] monitorTargets, boolean useRegExp,</span> |
| <span class="source-line-no">1471</span><span id="line-1471"> Sink sink, ExecutorService executor, boolean writeSniffing, TableName writeTableName,</span> |
| <span class="source-line-no">1472</span><span id="line-1472"> boolean treatFailureAsError, HashMap<String, Long> configuredReadTableTimeouts,</span> |
| <span class="source-line-no">1473</span><span id="line-1473"> long configuredWriteTableTimeout, long allowedFailures) {</span> |
| <span class="source-line-no">1474</span><span id="line-1474"> super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError,</span> |
| <span class="source-line-no">1475</span><span id="line-1475"> allowedFailures);</span> |
| <span class="source-line-no">1476</span><span id="line-1476"> Configuration conf = connection.getConfiguration();</span> |
| <span class="source-line-no">1477</span><span id="line-1477"> this.writeSniffing = writeSniffing;</span> |
| <span class="source-line-no">1478</span><span id="line-1478"> this.writeTableName = writeTableName;</span> |
| <span class="source-line-no">1479</span><span id="line-1479"> this.writeDataTTL =</span> |
| <span class="source-line-no">1480</span><span id="line-1480"> conf.getInt(HConstants.HBASE_CANARY_WRITE_DATA_TTL_KEY, DEFAULT_WRITE_DATA_TTL);</span> |
| <span class="source-line-no">1481</span><span id="line-1481"> this.regionsLowerLimit =</span> |
| <span class="source-line-no">1482</span><span id="line-1482"> conf.getFloat(HConstants.HBASE_CANARY_WRITE_PERSERVER_REGIONS_LOWERLIMIT_KEY, 1.0f);</span> |
| <span class="source-line-no">1483</span><span id="line-1483"> this.regionsUpperLimit =</span> |
| <span class="source-line-no">1484</span><span id="line-1484"> conf.getFloat(HConstants.HBASE_CANARY_WRITE_PERSERVER_REGIONS_UPPERLIMIT_KEY, 1.5f);</span> |
| <span class="source-line-no">1485</span><span id="line-1485"> this.checkPeriod = conf.getInt(HConstants.HBASE_CANARY_WRITE_TABLE_CHECK_PERIOD_KEY,</span> |
| <span class="source-line-no">1486</span><span id="line-1486"> DEFAULT_WRITE_TABLE_CHECK_PERIOD);</span> |
| <span class="source-line-no">1487</span><span id="line-1487"> this.rawScanEnabled = conf.getBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, false);</span> |
| <span class="source-line-no">1488</span><span id="line-1488"> this.configuredReadTableTimeouts = new HashMap<>(configuredReadTableTimeouts);</span> |
| <span class="source-line-no">1489</span><span id="line-1489"> this.configuredWriteTableTimeout = configuredWriteTableTimeout;</span> |
| <span class="source-line-no">1490</span><span id="line-1490"> this.readAllCF = conf.getBoolean(HConstants.HBASE_CANARY_READ_ALL_CF, true);</span> |
| <span class="source-line-no">1491</span><span id="line-1491"> }</span> |
| <span class="source-line-no">1492</span><span id="line-1492"></span> |
| <span class="source-line-no">1493</span><span id="line-1493"> private RegionStdOutSink getSink() {</span> |
| <span class="source-line-no">1494</span><span id="line-1494"> if (!(sink instanceof RegionStdOutSink)) {</span> |
| <span class="source-line-no">1495</span><span id="line-1495"> throw new RuntimeException("Can only write to Region sink");</span> |
| <span class="source-line-no">1496</span><span id="line-1496"> }</span> |
| <span class="source-line-no">1497</span><span id="line-1497"> return ((RegionStdOutSink) sink);</span> |
| <span class="source-line-no">1498</span><span id="line-1498"> }</span> |
| <span class="source-line-no">1499</span><span id="line-1499"></span> |
| <span class="source-line-no">1500</span><span id="line-1500"> @Override</span> |
| <span class="source-line-no">1501</span><span id="line-1501"> public void run() {</span> |
| <span class="source-line-no">1502</span><span id="line-1502"> if (this.initAdmin()) {</span> |
| <span class="source-line-no">1503</span><span id="line-1503"> try {</span> |
| <span class="source-line-no">1504</span><span id="line-1504"> List<Future<Void>> taskFutures = new LinkedList<>();</span> |
| <span class="source-line-no">1505</span><span id="line-1505"> RegionStdOutSink regionSink = this.getSink();</span> |
| <span class="source-line-no">1506</span><span id="line-1506"> regionSink.resetFailuresCountDetails();</span> |
| <span class="source-line-no">1507</span><span id="line-1507"> if (this.targets != null && this.targets.length > 0) {</span> |
| <span class="source-line-no">1508</span><span id="line-1508"> String[] tables = generateMonitorTables(this.targets);</span> |
| <span class="source-line-no">1509</span><span id="line-1509"> // Check to see that each table name passed in the -readTableTimeouts argument is also</span> |
| <span class="source-line-no">1510</span><span id="line-1510"> // passed as a monitor target.</span> |
| <span class="source-line-no">1511</span><span id="line-1511"> if (</span> |
| <span class="source-line-no">1512</span><span id="line-1512"> !new HashSet<>(Arrays.asList(tables))</span> |
| <span class="source-line-no">1513</span><span id="line-1513"> .containsAll(this.configuredReadTableTimeouts.keySet())</span> |
| <span class="source-line-no">1514</span><span id="line-1514"> ) {</span> |
| <span class="source-line-no">1515</span><span id="line-1515"> LOG.error("-readTableTimeouts can only specify read timeouts for monitor targets "</span> |
| <span class="source-line-no">1516</span><span id="line-1516"> + "passed via command line.");</span> |
| <span class="source-line-no">1517</span><span id="line-1517"> this.errorCode = USAGE_EXIT_CODE;</span> |
| <span class="source-line-no">1518</span><span id="line-1518"> return;</span> |
| <span class="source-line-no">1519</span><span id="line-1519"> }</span> |
| <span class="source-line-no">1520</span><span id="line-1520"> this.initialized = true;</span> |
| <span class="source-line-no">1521</span><span id="line-1521"> for (String table : tables) {</span> |
| <span class="source-line-no">1522</span><span id="line-1522"> LongAdder readLatency = regionSink.initializeAndGetReadLatencyForTable(table);</span> |
| <span class="source-line-no">1523</span><span id="line-1523"> taskFutures.addAll(CanaryTool.sniff(admin, regionSink, table, executor, TaskType.READ,</span> |
| <span class="source-line-no">1524</span><span id="line-1524"> this.rawScanEnabled, readLatency, readAllCF));</span> |
| <span class="source-line-no">1525</span><span id="line-1525"> }</span> |
| <span class="source-line-no">1526</span><span id="line-1526"> } else {</span> |
| <span class="source-line-no">1527</span><span id="line-1527"> taskFutures.addAll(sniff(TaskType.READ, regionSink));</span> |
| <span class="source-line-no">1528</span><span id="line-1528"> }</span> |
| <span class="source-line-no">1529</span><span id="line-1529"></span> |
| <span class="source-line-no">1530</span><span id="line-1530"> if (writeSniffing) {</span> |
| <span class="source-line-no">1531</span><span id="line-1531"> if (EnvironmentEdgeManager.currentTime() - lastCheckTime > checkPeriod) {</span> |
| <span class="source-line-no">1532</span><span id="line-1532"> try {</span> |
| <span class="source-line-no">1533</span><span id="line-1533"> checkWriteTableDistribution();</span> |
| <span class="source-line-no">1534</span><span id="line-1534"> } catch (IOException e) {</span> |
| <span class="source-line-no">1535</span><span id="line-1535"> LOG.error("Check canary table distribution failed!", e);</span> |
| <span class="source-line-no">1536</span><span id="line-1536"> }</span> |
| <span class="source-line-no">1537</span><span id="line-1537"> lastCheckTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1538</span><span id="line-1538"> }</span> |
| <span class="source-line-no">1539</span><span id="line-1539"> // sniff canary table with write operation</span> |
| <span class="source-line-no">1540</span><span id="line-1540"> regionSink.initializeWriteLatency();</span> |
| <span class="source-line-no">1541</span><span id="line-1541"> LongAdder writeTableLatency = regionSink.getWriteLatency();</span> |
| <span class="source-line-no">1542</span><span id="line-1542"> taskFutures</span> |
| <span class="source-line-no">1543</span><span id="line-1543"> .addAll(CanaryTool.sniff(admin, regionSink, admin.getDescriptor(writeTableName),</span> |
| <span class="source-line-no">1544</span><span id="line-1544"> executor, TaskType.WRITE, this.rawScanEnabled, writeTableLatency, readAllCF));</span> |
| <span class="source-line-no">1545</span><span id="line-1545"> }</span> |
| <span class="source-line-no">1546</span><span id="line-1546"></span> |
| <span class="source-line-no">1547</span><span id="line-1547"> for (Future<Void> future : taskFutures) {</span> |
| <span class="source-line-no">1548</span><span id="line-1548"> try {</span> |
| <span class="source-line-no">1549</span><span id="line-1549"> future.get();</span> |
| <span class="source-line-no">1550</span><span id="line-1550"> } catch (ExecutionException e) {</span> |
| <span class="source-line-no">1551</span><span id="line-1551"> LOG.error("Sniff region failed!", e);</span> |
| <span class="source-line-no">1552</span><span id="line-1552"> }</span> |
| <span class="source-line-no">1553</span><span id="line-1553"> }</span> |
| <span class="source-line-no">1554</span><span id="line-1554"> Map<String, LongAdder> actualReadTableLatency = regionSink.getReadLatencyMap();</span> |
| <span class="source-line-no">1555</span><span id="line-1555"> for (Map.Entry<String, Long> entry : configuredReadTableTimeouts.entrySet()) {</span> |
| <span class="source-line-no">1556</span><span id="line-1556"> String tableName = entry.getKey();</span> |
| <span class="source-line-no">1557</span><span id="line-1557"> if (actualReadTableLatency.containsKey(tableName)) {</span> |
| <span class="source-line-no">1558</span><span id="line-1558"> Long actual = actualReadTableLatency.get(tableName).longValue();</span> |
| <span class="source-line-no">1559</span><span id="line-1559"> Long configured = entry.getValue();</span> |
| <span class="source-line-no">1560</span><span id="line-1560"> if (actual > configured) {</span> |
| <span class="source-line-no">1561</span><span id="line-1561"> LOG.error("Read operation for {} took {}ms exceeded the configured read timeout."</span> |
| <span class="source-line-no">1562</span><span id="line-1562"> + "(Configured read timeout {}ms.", tableName, actual, configured);</span> |
| <span class="source-line-no">1563</span><span id="line-1563"> } else {</span> |
| <span class="source-line-no">1564</span><span id="line-1564"> LOG.info("Read operation for {} took {}ms (Configured read timeout {}ms.",</span> |
| <span class="source-line-no">1565</span><span id="line-1565"> tableName, actual, configured);</span> |
| <span class="source-line-no">1566</span><span id="line-1566"> }</span> |
| <span class="source-line-no">1567</span><span id="line-1567"> } else {</span> |
| <span class="source-line-no">1568</span><span id="line-1568"> LOG.error("Read operation for {} failed!", tableName);</span> |
| <span class="source-line-no">1569</span><span id="line-1569"> }</span> |
| <span class="source-line-no">1570</span><span id="line-1570"> }</span> |
| <span class="source-line-no">1571</span><span id="line-1571"> if (this.writeSniffing) {</span> |
| <span class="source-line-no">1572</span><span id="line-1572"> String writeTableStringName = this.writeTableName.getNameAsString();</span> |
| <span class="source-line-no">1573</span><span id="line-1573"> long actualWriteLatency = regionSink.getWriteLatency().longValue();</span> |
| <span class="source-line-no">1574</span><span id="line-1574"> LOG.info("Write operation for {} took {}ms. Configured write timeout {}ms.",</span> |
| <span class="source-line-no">1575</span><span id="line-1575"> writeTableStringName, actualWriteLatency, this.configuredWriteTableTimeout);</span> |
| <span class="source-line-no">1576</span><span id="line-1576"> // Check that the writeTable write operation latency does not exceed the configured</span> |
| <span class="source-line-no">1577</span><span id="line-1577"> // timeout.</span> |
| <span class="source-line-no">1578</span><span id="line-1578"> if (actualWriteLatency > this.configuredWriteTableTimeout) {</span> |
| <span class="source-line-no">1579</span><span id="line-1579"> LOG.error("Write operation for {} exceeded the configured write timeout.",</span> |
| <span class="source-line-no">1580</span><span id="line-1580"> writeTableStringName);</span> |
| <span class="source-line-no">1581</span><span id="line-1581"> }</span> |
| <span class="source-line-no">1582</span><span id="line-1582"> }</span> |
| <span class="source-line-no">1583</span><span id="line-1583"> } catch (Exception e) {</span> |
| <span class="source-line-no">1584</span><span id="line-1584"> LOG.error("Run regionMonitor failed", e);</span> |
| <span class="source-line-no">1585</span><span id="line-1585"> this.errorCode = ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1586</span><span id="line-1586"> } finally {</span> |
| <span class="source-line-no">1587</span><span id="line-1587"> this.done = true;</span> |
| <span class="source-line-no">1588</span><span id="line-1588"> }</span> |
| <span class="source-line-no">1589</span><span id="line-1589"> }</span> |
| <span class="source-line-no">1590</span><span id="line-1590"> this.done = true;</span> |
| <span class="source-line-no">1591</span><span id="line-1591"> }</span> |
| <span class="source-line-no">1592</span><span id="line-1592"></span> |
| <span class="source-line-no">1593</span><span id="line-1593"> /** Returns List of tables to use in test. */</span> |
| <span class="source-line-no">1594</span><span id="line-1594"> private String[] generateMonitorTables(String[] monitorTargets) throws IOException {</span> |
| <span class="source-line-no">1595</span><span id="line-1595"> String[] returnTables = null;</span> |
| <span class="source-line-no">1596</span><span id="line-1596"></span> |
| <span class="source-line-no">1597</span><span id="line-1597"> if (this.useRegExp) {</span> |
| <span class="source-line-no">1598</span><span id="line-1598"> Pattern pattern = null;</span> |
| <span class="source-line-no">1599</span><span id="line-1599"> List<TableDescriptor> tds = null;</span> |
| <span class="source-line-no">1600</span><span id="line-1600"> Set<String> tmpTables = new TreeSet<>();</span> |
| <span class="source-line-no">1601</span><span id="line-1601"> try {</span> |
| <span class="source-line-no">1602</span><span id="line-1602"> LOG.debug(String.format("reading list of tables"));</span> |
| <span class="source-line-no">1603</span><span id="line-1603"> tds = this.admin.listTableDescriptors(pattern);</span> |
| <span class="source-line-no">1604</span><span id="line-1604"> if (tds == null) {</span> |
| <span class="source-line-no">1605</span><span id="line-1605"> tds = Collections.emptyList();</span> |
| <span class="source-line-no">1606</span><span id="line-1606"> }</span> |
| <span class="source-line-no">1607</span><span id="line-1607"> for (String monitorTarget : monitorTargets) {</span> |
| <span class="source-line-no">1608</span><span id="line-1608"> pattern = Pattern.compile(monitorTarget);</span> |
| <span class="source-line-no">1609</span><span id="line-1609"> for (TableDescriptor td : tds) {</span> |
| <span class="source-line-no">1610</span><span id="line-1610"> if (pattern.matcher(td.getTableName().getNameAsString()).matches()) {</span> |
| <span class="source-line-no">1611</span><span id="line-1611"> tmpTables.add(td.getTableName().getNameAsString());</span> |
| <span class="source-line-no">1612</span><span id="line-1612"> }</span> |
| <span class="source-line-no">1613</span><span id="line-1613"> }</span> |
| <span class="source-line-no">1614</span><span id="line-1614"> }</span> |
| <span class="source-line-no">1615</span><span id="line-1615"> } catch (IOException e) {</span> |
| <span class="source-line-no">1616</span><span id="line-1616"> LOG.error("Communicate with admin failed", e);</span> |
| <span class="source-line-no">1617</span><span id="line-1617"> throw e;</span> |
| <span class="source-line-no">1618</span><span id="line-1618"> }</span> |
| <span class="source-line-no">1619</span><span id="line-1619"></span> |
| <span class="source-line-no">1620</span><span id="line-1620"> if (tmpTables.size() > 0) {</span> |
| <span class="source-line-no">1621</span><span id="line-1621"> returnTables = tmpTables.toArray(new String[tmpTables.size()]);</span> |
| <span class="source-line-no">1622</span><span id="line-1622"> } else {</span> |
| <span class="source-line-no">1623</span><span id="line-1623"> String msg = "No HTable found, tablePattern:" + Arrays.toString(monitorTargets);</span> |
| <span class="source-line-no">1624</span><span id="line-1624"> LOG.error(msg);</span> |
| <span class="source-line-no">1625</span><span id="line-1625"> this.errorCode = INIT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1626</span><span id="line-1626"> throw new TableNotFoundException(msg);</span> |
| <span class="source-line-no">1627</span><span id="line-1627"> }</span> |
| <span class="source-line-no">1628</span><span id="line-1628"> } else {</span> |
| <span class="source-line-no">1629</span><span id="line-1629"> returnTables = monitorTargets;</span> |
| <span class="source-line-no">1630</span><span id="line-1630"> }</span> |
| <span class="source-line-no">1631</span><span id="line-1631"></span> |
| <span class="source-line-no">1632</span><span id="line-1632"> return returnTables;</span> |
| <span class="source-line-no">1633</span><span id="line-1633"> }</span> |
| <span class="source-line-no">1634</span><span id="line-1634"></span> |
| <span class="source-line-no">1635</span><span id="line-1635"> /*</span> |
| <span class="source-line-no">1636</span><span id="line-1636"> * Canary entry point to monitor all the tables.</span> |
| <span class="source-line-no">1637</span><span id="line-1637"> */</span> |
| <span class="source-line-no">1638</span><span id="line-1638"> private List<Future<Void>> sniff(TaskType taskType, RegionStdOutSink regionSink)</span> |
| <span class="source-line-no">1639</span><span id="line-1639"> throws Exception {</span> |
| <span class="source-line-no">1640</span><span id="line-1640"> LOG.debug("Reading list of tables");</span> |
| <span class="source-line-no">1641</span><span id="line-1641"> List<Future<Void>> taskFutures = new LinkedList<>();</span> |
| <span class="source-line-no">1642</span><span id="line-1642"> for (TableDescriptor td : admin.listTableDescriptors()) {</span> |
| <span class="source-line-no">1643</span><span id="line-1643"> if (</span> |
| <span class="source-line-no">1644</span><span id="line-1644"> admin.tableExists(td.getTableName()) && admin.isTableEnabled(td.getTableName())</span> |
| <span class="source-line-no">1645</span><span id="line-1645"> && (!td.getTableName().equals(writeTableName))</span> |
| <span class="source-line-no">1646</span><span id="line-1646"> ) {</span> |
| <span class="source-line-no">1647</span><span id="line-1647"> LongAdder readLatency =</span> |
| <span class="source-line-no">1648</span><span id="line-1648"> regionSink.initializeAndGetReadLatencyForTable(td.getTableName().getNameAsString());</span> |
| <span class="source-line-no">1649</span><span id="line-1649"> taskFutures.addAll(CanaryTool.sniff(admin, sink, td, executor, taskType,</span> |
| <span class="source-line-no">1650</span><span id="line-1650"> this.rawScanEnabled, readLatency, readAllCF));</span> |
| <span class="source-line-no">1651</span><span id="line-1651"> }</span> |
| <span class="source-line-no">1652</span><span id="line-1652"> }</span> |
| <span class="source-line-no">1653</span><span id="line-1653"> return taskFutures;</span> |
| <span class="source-line-no">1654</span><span id="line-1654"> }</span> |
| <span class="source-line-no">1655</span><span id="line-1655"></span> |
| <span class="source-line-no">1656</span><span id="line-1656"> private void checkWriteTableDistribution() throws IOException {</span> |
| <span class="source-line-no">1657</span><span id="line-1657"> if (!admin.tableExists(writeTableName)) {</span> |
| <span class="source-line-no">1658</span><span id="line-1658"> int numberOfServers = admin.getRegionServers().size();</span> |
| <span class="source-line-no">1659</span><span id="line-1659"> if (numberOfServers == 0) {</span> |
| <span class="source-line-no">1660</span><span id="line-1660"> throw new IllegalStateException("No live regionservers");</span> |
| <span class="source-line-no">1661</span><span id="line-1661"> }</span> |
| <span class="source-line-no">1662</span><span id="line-1662"> createWriteTable(numberOfServers);</span> |
| <span class="source-line-no">1663</span><span id="line-1663"> }</span> |
| <span class="source-line-no">1664</span><span id="line-1664"></span> |
| <span class="source-line-no">1665</span><span id="line-1665"> if (!admin.isTableEnabled(writeTableName)) {</span> |
| <span class="source-line-no">1666</span><span id="line-1666"> admin.enableTable(writeTableName);</span> |
| <span class="source-line-no">1667</span><span id="line-1667"> }</span> |
| <span class="source-line-no">1668</span><span id="line-1668"></span> |
| <span class="source-line-no">1669</span><span id="line-1669"> ClusterMetrics status =</span> |
| <span class="source-line-no">1670</span><span id="line-1670"> admin.getClusterMetrics(EnumSet.of(Option.SERVERS_NAME, Option.MASTER));</span> |
| <span class="source-line-no">1671</span><span id="line-1671"> int numberOfServers = status.getServersName().size();</span> |
| <span class="source-line-no">1672</span><span id="line-1672"> if (status.getServersName().contains(status.getMasterName())) {</span> |
| <span class="source-line-no">1673</span><span id="line-1673"> numberOfServers -= 1;</span> |
| <span class="source-line-no">1674</span><span id="line-1674"> }</span> |
| <span class="source-line-no">1675</span><span id="line-1675"></span> |
| <span class="source-line-no">1676</span><span id="line-1676"> List<Pair<RegionInfo, ServerName>> pairs =</span> |
| <span class="source-line-no">1677</span><span id="line-1677"> MetaTableAccessor.getTableRegionsAndLocations(connection, writeTableName);</span> |
| <span class="source-line-no">1678</span><span id="line-1678"> int numberOfRegions = pairs.size();</span> |
| <span class="source-line-no">1679</span><span id="line-1679"> if (</span> |
| <span class="source-line-no">1680</span><span id="line-1680"> numberOfRegions < numberOfServers * regionsLowerLimit</span> |
| <span class="source-line-no">1681</span><span id="line-1681"> || numberOfRegions > numberOfServers * regionsUpperLimit</span> |
| <span class="source-line-no">1682</span><span id="line-1682"> ) {</span> |
| <span class="source-line-no">1683</span><span id="line-1683"> admin.disableTable(writeTableName);</span> |
| <span class="source-line-no">1684</span><span id="line-1684"> admin.deleteTable(writeTableName);</span> |
| <span class="source-line-no">1685</span><span id="line-1685"> createWriteTable(numberOfServers);</span> |
| <span class="source-line-no">1686</span><span id="line-1686"> }</span> |
| <span class="source-line-no">1687</span><span id="line-1687"> HashSet<ServerName> serverSet = new HashSet<>();</span> |
| <span class="source-line-no">1688</span><span id="line-1688"> for (Pair<RegionInfo, ServerName> pair : pairs) {</span> |
| <span class="source-line-no">1689</span><span id="line-1689"> serverSet.add(pair.getSecond());</span> |
| <span class="source-line-no">1690</span><span id="line-1690"> }</span> |
| <span class="source-line-no">1691</span><span id="line-1691"> int numberOfCoveredServers = serverSet.size();</span> |
| <span class="source-line-no">1692</span><span id="line-1692"> if (numberOfCoveredServers < numberOfServers) {</span> |
| <span class="source-line-no">1693</span><span id="line-1693"> admin.balance();</span> |
| <span class="source-line-no">1694</span><span id="line-1694"> }</span> |
| <span class="source-line-no">1695</span><span id="line-1695"> }</span> |
| <span class="source-line-no">1696</span><span id="line-1696"></span> |
| <span class="source-line-no">1697</span><span id="line-1697"> private void createWriteTable(int numberOfServers) throws IOException {</span> |
| <span class="source-line-no">1698</span><span id="line-1698"> int numberOfRegions = (int) (numberOfServers * regionsLowerLimit);</span> |
| <span class="source-line-no">1699</span><span id="line-1699"> LOG.info("Number of live regionservers {}, pre-splitting the canary table into {} regions "</span> |
| <span class="source-line-no">1700</span><span id="line-1700"> + "(current lower limit of regions per server is {} and you can change it with config {}).",</span> |
| <span class="source-line-no">1701</span><span id="line-1701"> numberOfServers, numberOfRegions, regionsLowerLimit,</span> |
| <span class="source-line-no">1702</span><span id="line-1702"> HConstants.HBASE_CANARY_WRITE_PERSERVER_REGIONS_LOWERLIMIT_KEY);</span> |
| <span class="source-line-no">1703</span><span id="line-1703"> ColumnFamilyDescriptor family =</span> |
| <span class="source-line-no">1704</span><span id="line-1704"> ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CANARY_TABLE_FAMILY_NAME))</span> |
| <span class="source-line-no">1705</span><span id="line-1705"> .setMaxVersions(1).setTimeToLive(writeDataTTL).build();</span> |
| <span class="source-line-no">1706</span><span id="line-1706"> TableDescriptor desc =</span> |
| <span class="source-line-no">1707</span><span id="line-1707"> TableDescriptorBuilder.newBuilder(writeTableName).setColumnFamily(family).build();</span> |
| <span class="source-line-no">1708</span><span id="line-1708"> byte[][] splits = new RegionSplitter.HexStringSplit().split(numberOfRegions);</span> |
| <span class="source-line-no">1709</span><span id="line-1709"> admin.createTable(desc, splits);</span> |
| <span class="source-line-no">1710</span><span id="line-1710"> }</span> |
| <span class="source-line-no">1711</span><span id="line-1711"> }</span> |
| <span class="source-line-no">1712</span><span id="line-1712"></span> |
| <span class="source-line-no">1713</span><span id="line-1713"> /**</span> |
| <span class="source-line-no">1714</span><span id="line-1714"> * Canary entry point for specified table.</span> |
| <span class="source-line-no">1715</span><span id="line-1715"> * @throws Exception exception</span> |
| <span class="source-line-no">1716</span><span id="line-1716"> */</span> |
| <span class="source-line-no">1717</span><span id="line-1717"> private static List<Future<Void>> sniff(final Admin admin, final Sink sink, String tableName,</span> |
| <span class="source-line-no">1718</span><span id="line-1718"> ExecutorService executor, TaskType taskType, boolean rawScanEnabled, LongAdder readLatency,</span> |
| <span class="source-line-no">1719</span><span id="line-1719"> boolean readAllCF) throws Exception {</span> |
| <span class="source-line-no">1720</span><span id="line-1720"> LOG.debug("Checking table is enabled and getting table descriptor for table {}", tableName);</span> |
| <span class="source-line-no">1721</span><span id="line-1721"> if (admin.isTableEnabled(TableName.valueOf(tableName))) {</span> |
| <span class="source-line-no">1722</span><span id="line-1722"> return CanaryTool.sniff(admin, sink, admin.getDescriptor(TableName.valueOf(tableName)),</span> |
| <span class="source-line-no">1723</span><span id="line-1723"> executor, taskType, rawScanEnabled, readLatency, readAllCF);</span> |
| <span class="source-line-no">1724</span><span id="line-1724"> } else {</span> |
| <span class="source-line-no">1725</span><span id="line-1725"> LOG.warn("Table {} is not enabled", tableName);</span> |
| <span class="source-line-no">1726</span><span id="line-1726"> }</span> |
| <span class="source-line-no">1727</span><span id="line-1727"> return new LinkedList<>();</span> |
| <span class="source-line-no">1728</span><span id="line-1728"> }</span> |
| <span class="source-line-no">1729</span><span id="line-1729"></span> |
| <span class="source-line-no">1730</span><span id="line-1730"> /*</span> |
| <span class="source-line-no">1731</span><span id="line-1731"> * Loops over regions of this table, and outputs information about the state.</span> |
| <span class="source-line-no">1732</span><span id="line-1732"> */</span> |
| <span class="source-line-no">1733</span><span id="line-1733"> private static List<Future<Void>> sniff(final Admin admin, final Sink sink,</span> |
| <span class="source-line-no">1734</span><span id="line-1734"> TableDescriptor tableDesc, ExecutorService executor, TaskType taskType, boolean rawScanEnabled,</span> |
| <span class="source-line-no">1735</span><span id="line-1735"> LongAdder rwLatency, boolean readAllCF) throws Exception {</span> |
| <span class="source-line-no">1736</span><span id="line-1736"> LOG.debug("Reading list of regions for table {}", tableDesc.getTableName());</span> |
| <span class="source-line-no">1737</span><span id="line-1737"> try (Table table = admin.getConnection().getTable(tableDesc.getTableName())) {</span> |
| <span class="source-line-no">1738</span><span id="line-1738"> List<RegionTask> tasks = new ArrayList<>();</span> |
| <span class="source-line-no">1739</span><span id="line-1739"> try (RegionLocator regionLocator =</span> |
| <span class="source-line-no">1740</span><span id="line-1740"> admin.getConnection().getRegionLocator(tableDesc.getTableName())) {</span> |
| <span class="source-line-no">1741</span><span id="line-1741"> for (HRegionLocation location : regionLocator.getAllRegionLocations()) {</span> |
| <span class="source-line-no">1742</span><span id="line-1742"> if (location == null) {</span> |
| <span class="source-line-no">1743</span><span id="line-1743"> LOG.warn("Null location");</span> |
| <span class="source-line-no">1744</span><span id="line-1744"> continue;</span> |
| <span class="source-line-no">1745</span><span id="line-1745"> }</span> |
| <span class="source-line-no">1746</span><span id="line-1746"> ServerName rs = location.getServerName();</span> |
| <span class="source-line-no">1747</span><span id="line-1747"> RegionInfo region = location.getRegion();</span> |
| <span class="source-line-no">1748</span><span id="line-1748"> tasks.add(new RegionTask(admin.getConnection(), region, rs, (RegionStdOutSink) sink,</span> |
| <span class="source-line-no">1749</span><span id="line-1749"> taskType, rawScanEnabled, rwLatency, readAllCF));</span> |
| <span class="source-line-no">1750</span><span id="line-1750"> Map<String, List<RegionTaskResult>> regionMap = ((RegionStdOutSink) sink).getRegionMap();</span> |
| <span class="source-line-no">1751</span><span id="line-1751"> regionMap.put(region.getRegionNameAsString(), new ArrayList<RegionTaskResult>());</span> |
| <span class="source-line-no">1752</span><span id="line-1752"> }</span> |
| <span class="source-line-no">1753</span><span id="line-1753"> return executor.invokeAll(tasks);</span> |
| <span class="source-line-no">1754</span><span id="line-1754"> }</span> |
| <span class="source-line-no">1755</span><span id="line-1755"> } catch (TableNotFoundException e) {</span> |
| <span class="source-line-no">1756</span><span id="line-1756"> return Collections.EMPTY_LIST;</span> |
| <span class="source-line-no">1757</span><span id="line-1757"> }</span> |
| <span class="source-line-no">1758</span><span id="line-1758"> }</span> |
| <span class="source-line-no">1759</span><span id="line-1759"></span> |
| <span class="source-line-no">1760</span><span id="line-1760"> // monitor for zookeeper mode</span> |
| <span class="source-line-no">1761</span><span id="line-1761"> private static class ZookeeperMonitor extends Monitor {</span> |
| <span class="source-line-no">1762</span><span id="line-1762"> private List<String> hosts;</span> |
| <span class="source-line-no">1763</span><span id="line-1763"> private final String znode;</span> |
| <span class="source-line-no">1764</span><span id="line-1764"> private final int timeout;</span> |
| <span class="source-line-no">1765</span><span id="line-1765"></span> |
| <span class="source-line-no">1766</span><span id="line-1766"> protected ZookeeperMonitor(Connection connection, String[] monitorTargets, boolean useRegExp,</span> |
| <span class="source-line-no">1767</span><span id="line-1767"> Sink sink, ExecutorService executor, boolean treatFailureAsError, long allowedFailures) {</span> |
| <span class="source-line-no">1768</span><span id="line-1768"> super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError,</span> |
| <span class="source-line-no">1769</span><span id="line-1769"> allowedFailures);</span> |
| <span class="source-line-no">1770</span><span id="line-1770"> Configuration configuration = connection.getConfiguration();</span> |
| <span class="source-line-no">1771</span><span id="line-1771"> znode = configuration.get(ZOOKEEPER_ZNODE_PARENT, DEFAULT_ZOOKEEPER_ZNODE_PARENT);</span> |
| <span class="source-line-no">1772</span><span id="line-1772"> timeout =</span> |
| <span class="source-line-no">1773</span><span id="line-1773"> configuration.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);</span> |
| <span class="source-line-no">1774</span><span id="line-1774"> ConnectStringParser parser =</span> |
| <span class="source-line-no">1775</span><span id="line-1775"> new ConnectStringParser(ZKConfig.getZKQuorumServersString(configuration));</span> |
| <span class="source-line-no">1776</span><span id="line-1776"> hosts = Lists.newArrayList();</span> |
| <span class="source-line-no">1777</span><span id="line-1777"> for (InetSocketAddress server : parser.getServerAddresses()) {</span> |
| <span class="source-line-no">1778</span><span id="line-1778"> hosts.add(inetSocketAddress2String(server));</span> |
| <span class="source-line-no">1779</span><span id="line-1779"> }</span> |
| <span class="source-line-no">1780</span><span id="line-1780"> if (allowedFailures > (hosts.size() - 1) / 2) {</span> |
| <span class="source-line-no">1781</span><span id="line-1781"> LOG.warn(</span> |
| <span class="source-line-no">1782</span><span id="line-1782"> "Confirm allowable number of failed ZooKeeper nodes, as quorum will "</span> |
| <span class="source-line-no">1783</span><span id="line-1783"> + "already be lost. Setting of {} failures is unexpected for {} ensemble size.",</span> |
| <span class="source-line-no">1784</span><span id="line-1784"> allowedFailures, hosts.size());</span> |
| <span class="source-line-no">1785</span><span id="line-1785"> }</span> |
| <span class="source-line-no">1786</span><span id="line-1786"> }</span> |
| <span class="source-line-no">1787</span><span id="line-1787"></span> |
| <span class="source-line-no">1788</span><span id="line-1788"> @Override</span> |
| <span class="source-line-no">1789</span><span id="line-1789"> public void run() {</span> |
| <span class="source-line-no">1790</span><span id="line-1790"> List<ZookeeperTask> tasks = Lists.newArrayList();</span> |
| <span class="source-line-no">1791</span><span id="line-1791"> ZookeeperStdOutSink zkSink = null;</span> |
| <span class="source-line-no">1792</span><span id="line-1792"> try {</span> |
| <span class="source-line-no">1793</span><span id="line-1793"> zkSink = this.getSink();</span> |
| <span class="source-line-no">1794</span><span id="line-1794"> } catch (RuntimeException e) {</span> |
| <span class="source-line-no">1795</span><span id="line-1795"> LOG.error("Run ZooKeeperMonitor failed!", e);</span> |
| <span class="source-line-no">1796</span><span id="line-1796"> this.errorCode = ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1797</span><span id="line-1797"> }</span> |
| <span class="source-line-no">1798</span><span id="line-1798"> this.initialized = true;</span> |
| <span class="source-line-no">1799</span><span id="line-1799"> for (final String host : hosts) {</span> |
| <span class="source-line-no">1800</span><span id="line-1800"> tasks.add(new ZookeeperTask(connection, host, znode, timeout, zkSink));</span> |
| <span class="source-line-no">1801</span><span id="line-1801"> }</span> |
| <span class="source-line-no">1802</span><span id="line-1802"> try {</span> |
| <span class="source-line-no">1803</span><span id="line-1803"> for (Future<Void> future : this.executor.invokeAll(tasks)) {</span> |
| <span class="source-line-no">1804</span><span id="line-1804"> try {</span> |
| <span class="source-line-no">1805</span><span id="line-1805"> future.get();</span> |
| <span class="source-line-no">1806</span><span id="line-1806"> } catch (ExecutionException e) {</span> |
| <span class="source-line-no">1807</span><span id="line-1807"> LOG.error("Sniff zookeeper failed!", e);</span> |
| <span class="source-line-no">1808</span><span id="line-1808"> this.errorCode = ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1809</span><span id="line-1809"> }</span> |
| <span class="source-line-no">1810</span><span id="line-1810"> }</span> |
| <span class="source-line-no">1811</span><span id="line-1811"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">1812</span><span id="line-1812"> this.errorCode = ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1813</span><span id="line-1813"> Thread.currentThread().interrupt();</span> |
| <span class="source-line-no">1814</span><span id="line-1814"> LOG.error("Sniff zookeeper interrupted!", e);</span> |
| <span class="source-line-no">1815</span><span id="line-1815"> }</span> |
| <span class="source-line-no">1816</span><span id="line-1816"> this.done = true;</span> |
| <span class="source-line-no">1817</span><span id="line-1817"> }</span> |
| <span class="source-line-no">1818</span><span id="line-1818"></span> |
| <span class="source-line-no">1819</span><span id="line-1819"> private ZookeeperStdOutSink getSink() {</span> |
| <span class="source-line-no">1820</span><span id="line-1820"> if (!(sink instanceof ZookeeperStdOutSink)) {</span> |
| <span class="source-line-no">1821</span><span id="line-1821"> throw new RuntimeException("Can only write to zookeeper sink");</span> |
| <span class="source-line-no">1822</span><span id="line-1822"> }</span> |
| <span class="source-line-no">1823</span><span id="line-1823"> return ((ZookeeperStdOutSink) sink);</span> |
| <span class="source-line-no">1824</span><span id="line-1824"> }</span> |
| <span class="source-line-no">1825</span><span id="line-1825"> }</span> |
| <span class="source-line-no">1826</span><span id="line-1826"></span> |
| <span class="source-line-no">1827</span><span id="line-1827"> /**</span> |
| <span class="source-line-no">1828</span><span id="line-1828"> * A monitor for regionserver mode</span> |
| <span class="source-line-no">1829</span><span id="line-1829"> */</span> |
| <span class="source-line-no">1830</span><span id="line-1830"> private static class RegionServerMonitor extends Monitor {</span> |
| <span class="source-line-no">1831</span><span id="line-1831"> private boolean rawScanEnabled;</span> |
| <span class="source-line-no">1832</span><span id="line-1832"> private boolean allRegions;</span> |
| <span class="source-line-no">1833</span><span id="line-1833"></span> |
| <span class="source-line-no">1834</span><span id="line-1834"> public RegionServerMonitor(Connection connection, String[] monitorTargets, boolean useRegExp,</span> |
| <span class="source-line-no">1835</span><span id="line-1835"> Sink sink, ExecutorService executor, boolean allRegions, boolean treatFailureAsError,</span> |
| <span class="source-line-no">1836</span><span id="line-1836"> long allowedFailures) {</span> |
| <span class="source-line-no">1837</span><span id="line-1837"> super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError,</span> |
| <span class="source-line-no">1838</span><span id="line-1838"> allowedFailures);</span> |
| <span class="source-line-no">1839</span><span id="line-1839"> Configuration conf = connection.getConfiguration();</span> |
| <span class="source-line-no">1840</span><span id="line-1840"> this.rawScanEnabled = conf.getBoolean(HConstants.HBASE_CANARY_READ_RAW_SCAN_KEY, false);</span> |
| <span class="source-line-no">1841</span><span id="line-1841"> this.allRegions = allRegions;</span> |
| <span class="source-line-no">1842</span><span id="line-1842"> }</span> |
| <span class="source-line-no">1843</span><span id="line-1843"></span> |
| <span class="source-line-no">1844</span><span id="line-1844"> private RegionServerStdOutSink getSink() {</span> |
| <span class="source-line-no">1845</span><span id="line-1845"> if (!(sink instanceof RegionServerStdOutSink)) {</span> |
| <span class="source-line-no">1846</span><span id="line-1846"> throw new RuntimeException("Can only write to regionserver sink");</span> |
| <span class="source-line-no">1847</span><span id="line-1847"> }</span> |
| <span class="source-line-no">1848</span><span id="line-1848"> return ((RegionServerStdOutSink) sink);</span> |
| <span class="source-line-no">1849</span><span id="line-1849"> }</span> |
| <span class="source-line-no">1850</span><span id="line-1850"></span> |
| <span class="source-line-no">1851</span><span id="line-1851"> @Override</span> |
| <span class="source-line-no">1852</span><span id="line-1852"> public void run() {</span> |
| <span class="source-line-no">1853</span><span id="line-1853"> if (this.initAdmin() && this.checkNoTableNames()) {</span> |
| <span class="source-line-no">1854</span><span id="line-1854"> RegionServerStdOutSink regionServerSink = null;</span> |
| <span class="source-line-no">1855</span><span id="line-1855"> try {</span> |
| <span class="source-line-no">1856</span><span id="line-1856"> regionServerSink = this.getSink();</span> |
| <span class="source-line-no">1857</span><span id="line-1857"> } catch (RuntimeException e) {</span> |
| <span class="source-line-no">1858</span><span id="line-1858"> LOG.error("Run RegionServerMonitor failed!", e);</span> |
| <span class="source-line-no">1859</span><span id="line-1859"> this.errorCode = ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1860</span><span id="line-1860"> }</span> |
| <span class="source-line-no">1861</span><span id="line-1861"> Map<String, List<RegionInfo>> rsAndRMap = this.filterRegionServerByName();</span> |
| <span class="source-line-no">1862</span><span id="line-1862"> this.initialized = true;</span> |
| <span class="source-line-no">1863</span><span id="line-1863"> this.monitorRegionServers(rsAndRMap, regionServerSink);</span> |
| <span class="source-line-no">1864</span><span id="line-1864"> }</span> |
| <span class="source-line-no">1865</span><span id="line-1865"> this.done = true;</span> |
| <span class="source-line-no">1866</span><span id="line-1866"> }</span> |
| <span class="source-line-no">1867</span><span id="line-1867"></span> |
| <span class="source-line-no">1868</span><span id="line-1868"> private boolean checkNoTableNames() {</span> |
| <span class="source-line-no">1869</span><span id="line-1869"> List<String> foundTableNames = new ArrayList<>();</span> |
| <span class="source-line-no">1870</span><span id="line-1870"> TableName[] tableNames = null;</span> |
| <span class="source-line-no">1871</span><span id="line-1871"> LOG.debug("Reading list of tables");</span> |
| <span class="source-line-no">1872</span><span id="line-1872"> try {</span> |
| <span class="source-line-no">1873</span><span id="line-1873"> tableNames = this.admin.listTableNames();</span> |
| <span class="source-line-no">1874</span><span id="line-1874"> } catch (IOException e) {</span> |
| <span class="source-line-no">1875</span><span id="line-1875"> LOG.error("Get listTableNames failed", e);</span> |
| <span class="source-line-no">1876</span><span id="line-1876"> this.errorCode = INIT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1877</span><span id="line-1877"> return false;</span> |
| <span class="source-line-no">1878</span><span id="line-1878"> }</span> |
| <span class="source-line-no">1879</span><span id="line-1879"></span> |
| <span class="source-line-no">1880</span><span id="line-1880"> if (this.targets == null || this.targets.length == 0) {</span> |
| <span class="source-line-no">1881</span><span id="line-1881"> return true;</span> |
| <span class="source-line-no">1882</span><span id="line-1882"> }</span> |
| <span class="source-line-no">1883</span><span id="line-1883"></span> |
| <span class="source-line-no">1884</span><span id="line-1884"> for (String target : this.targets) {</span> |
| <span class="source-line-no">1885</span><span id="line-1885"> for (TableName tableName : tableNames) {</span> |
| <span class="source-line-no">1886</span><span id="line-1886"> if (target.equals(tableName.getNameAsString())) {</span> |
| <span class="source-line-no">1887</span><span id="line-1887"> foundTableNames.add(target);</span> |
| <span class="source-line-no">1888</span><span id="line-1888"> }</span> |
| <span class="source-line-no">1889</span><span id="line-1889"> }</span> |
| <span class="source-line-no">1890</span><span id="line-1890"> }</span> |
| <span class="source-line-no">1891</span><span id="line-1891"></span> |
| <span class="source-line-no">1892</span><span id="line-1892"> if (foundTableNames.size() > 0) {</span> |
| <span class="source-line-no">1893</span><span id="line-1893"> System.err.println("Cannot pass a tablename when using the -regionserver "</span> |
| <span class="source-line-no">1894</span><span id="line-1894"> + "option, tablenames:" + foundTableNames.toString());</span> |
| <span class="source-line-no">1895</span><span id="line-1895"> this.errorCode = USAGE_EXIT_CODE;</span> |
| <span class="source-line-no">1896</span><span id="line-1896"> }</span> |
| <span class="source-line-no">1897</span><span id="line-1897"> return foundTableNames.isEmpty();</span> |
| <span class="source-line-no">1898</span><span id="line-1898"> }</span> |
| <span class="source-line-no">1899</span><span id="line-1899"></span> |
| <span class="source-line-no">1900</span><span id="line-1900"> private void monitorRegionServers(Map<String, List<RegionInfo>> rsAndRMap,</span> |
| <span class="source-line-no">1901</span><span id="line-1901"> RegionServerStdOutSink regionServerSink) {</span> |
| <span class="source-line-no">1902</span><span id="line-1902"> List<RegionServerTask> tasks = new ArrayList<>();</span> |
| <span class="source-line-no">1903</span><span id="line-1903"> Map<String, AtomicLong> successMap = new HashMap<>();</span> |
| <span class="source-line-no">1904</span><span id="line-1904"> for (Map.Entry<String, List<RegionInfo>> entry : rsAndRMap.entrySet()) {</span> |
| <span class="source-line-no">1905</span><span id="line-1905"> String serverName = entry.getKey();</span> |
| <span class="source-line-no">1906</span><span id="line-1906"> AtomicLong successes = new AtomicLong(0);</span> |
| <span class="source-line-no">1907</span><span id="line-1907"> successMap.put(serverName, successes);</span> |
| <span class="source-line-no">1908</span><span id="line-1908"> if (entry.getValue().isEmpty()) {</span> |
| <span class="source-line-no">1909</span><span id="line-1909"> LOG.error("Regionserver not serving any regions - {}", serverName);</span> |
| <span class="source-line-no">1910</span><span id="line-1910"> } else if (this.allRegions) {</span> |
| <span class="source-line-no">1911</span><span id="line-1911"> for (RegionInfo region : entry.getValue()) {</span> |
| <span class="source-line-no">1912</span><span id="line-1912"> tasks.add(new RegionServerTask(this.connection, serverName, region, regionServerSink,</span> |
| <span class="source-line-no">1913</span><span id="line-1913"> this.rawScanEnabled, successes));</span> |
| <span class="source-line-no">1914</span><span id="line-1914"> }</span> |
| <span class="source-line-no">1915</span><span id="line-1915"> } else {</span> |
| <span class="source-line-no">1916</span><span id="line-1916"> // random select a region if flag not set</span> |
| <span class="source-line-no">1917</span><span id="line-1917"> RegionInfo region =</span> |
| <span class="source-line-no">1918</span><span id="line-1918"> entry.getValue().get(ThreadLocalRandom.current().nextInt(entry.getValue().size()));</span> |
| <span class="source-line-no">1919</span><span id="line-1919"> tasks.add(new RegionServerTask(this.connection, serverName, region, regionServerSink,</span> |
| <span class="source-line-no">1920</span><span id="line-1920"> this.rawScanEnabled, successes));</span> |
| <span class="source-line-no">1921</span><span id="line-1921"> }</span> |
| <span class="source-line-no">1922</span><span id="line-1922"> }</span> |
| <span class="source-line-no">1923</span><span id="line-1923"> try {</span> |
| <span class="source-line-no">1924</span><span id="line-1924"> for (Future<Void> future : this.executor.invokeAll(tasks)) {</span> |
| <span class="source-line-no">1925</span><span id="line-1925"> try {</span> |
| <span class="source-line-no">1926</span><span id="line-1926"> future.get();</span> |
| <span class="source-line-no">1927</span><span id="line-1927"> } catch (ExecutionException e) {</span> |
| <span class="source-line-no">1928</span><span id="line-1928"> LOG.error("Sniff regionserver failed!", e);</span> |
| <span class="source-line-no">1929</span><span id="line-1929"> this.errorCode = ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1930</span><span id="line-1930"> }</span> |
| <span class="source-line-no">1931</span><span id="line-1931"> }</span> |
| <span class="source-line-no">1932</span><span id="line-1932"> if (this.allRegions) {</span> |
| <span class="source-line-no">1933</span><span id="line-1933"> for (Map.Entry<String, List<RegionInfo>> entry : rsAndRMap.entrySet()) {</span> |
| <span class="source-line-no">1934</span><span id="line-1934"> String serverName = entry.getKey();</span> |
| <span class="source-line-no">1935</span><span id="line-1935"> LOG.info("Successfully read {} regions out of {} on regionserver {}",</span> |
| <span class="source-line-no">1936</span><span id="line-1936"> successMap.get(serverName), entry.getValue().size(), serverName);</span> |
| <span class="source-line-no">1937</span><span id="line-1937"> }</span> |
| <span class="source-line-no">1938</span><span id="line-1938"> }</span> |
| <span class="source-line-no">1939</span><span id="line-1939"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">1940</span><span id="line-1940"> this.errorCode = ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1941</span><span id="line-1941"> LOG.error("Sniff regionserver interrupted!", e);</span> |
| <span class="source-line-no">1942</span><span id="line-1942"> }</span> |
| <span class="source-line-no">1943</span><span id="line-1943"> }</span> |
| <span class="source-line-no">1944</span><span id="line-1944"></span> |
| <span class="source-line-no">1945</span><span id="line-1945"> private Map<String, List<RegionInfo>> filterRegionServerByName() {</span> |
| <span class="source-line-no">1946</span><span id="line-1946"> Map<String, List<RegionInfo>> regionServerAndRegionsMap = this.getAllRegionServerByName();</span> |
| <span class="source-line-no">1947</span><span id="line-1947"> regionServerAndRegionsMap = this.doFilterRegionServerByName(regionServerAndRegionsMap);</span> |
| <span class="source-line-no">1948</span><span id="line-1948"> return regionServerAndRegionsMap;</span> |
| <span class="source-line-no">1949</span><span id="line-1949"> }</span> |
| <span class="source-line-no">1950</span><span id="line-1950"></span> |
| <span class="source-line-no">1951</span><span id="line-1951"> private Map<String, List<RegionInfo>> getAllRegionServerByName() {</span> |
| <span class="source-line-no">1952</span><span id="line-1952"> Map<String, List<RegionInfo>> rsAndRMap = new HashMap<>();</span> |
| <span class="source-line-no">1953</span><span id="line-1953"> try {</span> |
| <span class="source-line-no">1954</span><span id="line-1954"> LOG.debug("Reading list of tables and locations");</span> |
| <span class="source-line-no">1955</span><span id="line-1955"> List<TableDescriptor> tableDescs = this.admin.listTableDescriptors();</span> |
| <span class="source-line-no">1956</span><span id="line-1956"> List<RegionInfo> regions = null;</span> |
| <span class="source-line-no">1957</span><span id="line-1957"> for (TableDescriptor tableDesc : tableDescs) {</span> |
| <span class="source-line-no">1958</span><span id="line-1958"> try (RegionLocator regionLocator =</span> |
| <span class="source-line-no">1959</span><span id="line-1959"> this.admin.getConnection().getRegionLocator(tableDesc.getTableName())) {</span> |
| <span class="source-line-no">1960</span><span id="line-1960"> for (HRegionLocation location : regionLocator.getAllRegionLocations()) {</span> |
| <span class="source-line-no">1961</span><span id="line-1961"> if (location == null) {</span> |
| <span class="source-line-no">1962</span><span id="line-1962"> LOG.warn("Null location");</span> |
| <span class="source-line-no">1963</span><span id="line-1963"> continue;</span> |
| <span class="source-line-no">1964</span><span id="line-1964"> }</span> |
| <span class="source-line-no">1965</span><span id="line-1965"> ServerName rs = location.getServerName();</span> |
| <span class="source-line-no">1966</span><span id="line-1966"> String rsName = rs.getHostname();</span> |
| <span class="source-line-no">1967</span><span id="line-1967"> RegionInfo r = location.getRegion();</span> |
| <span class="source-line-no">1968</span><span id="line-1968"> if (rsAndRMap.containsKey(rsName)) {</span> |
| <span class="source-line-no">1969</span><span id="line-1969"> regions = rsAndRMap.get(rsName);</span> |
| <span class="source-line-no">1970</span><span id="line-1970"> } else {</span> |
| <span class="source-line-no">1971</span><span id="line-1971"> regions = new ArrayList<>();</span> |
| <span class="source-line-no">1972</span><span id="line-1972"> rsAndRMap.put(rsName, regions);</span> |
| <span class="source-line-no">1973</span><span id="line-1973"> }</span> |
| <span class="source-line-no">1974</span><span id="line-1974"> regions.add(r);</span> |
| <span class="source-line-no">1975</span><span id="line-1975"> }</span> |
| <span class="source-line-no">1976</span><span id="line-1976"> }</span> |
| <span class="source-line-no">1977</span><span id="line-1977"> }</span> |
| <span class="source-line-no">1978</span><span id="line-1978"></span> |
| <span class="source-line-no">1979</span><span id="line-1979"> // get any live regionservers not serving any regions</span> |
| <span class="source-line-no">1980</span><span id="line-1980"> for (ServerName rs : this.admin.getRegionServers()) {</span> |
| <span class="source-line-no">1981</span><span id="line-1981"> String rsName = rs.getHostname();</span> |
| <span class="source-line-no">1982</span><span id="line-1982"> if (!rsAndRMap.containsKey(rsName)) {</span> |
| <span class="source-line-no">1983</span><span id="line-1983"> rsAndRMap.put(rsName, Collections.<RegionInfo> emptyList());</span> |
| <span class="source-line-no">1984</span><span id="line-1984"> }</span> |
| <span class="source-line-no">1985</span><span id="line-1985"> }</span> |
| <span class="source-line-no">1986</span><span id="line-1986"> } catch (IOException e) {</span> |
| <span class="source-line-no">1987</span><span id="line-1987"> LOG.error("Get HTables info failed", e);</span> |
| <span class="source-line-no">1988</span><span id="line-1988"> this.errorCode = INIT_ERROR_EXIT_CODE;</span> |
| <span class="source-line-no">1989</span><span id="line-1989"> }</span> |
| <span class="source-line-no">1990</span><span id="line-1990"> return rsAndRMap;</span> |
| <span class="source-line-no">1991</span><span id="line-1991"> }</span> |
| <span class="source-line-no">1992</span><span id="line-1992"></span> |
| <span class="source-line-no">1993</span><span id="line-1993"> private Map<String, List<RegionInfo>></span> |
| <span class="source-line-no">1994</span><span id="line-1994"> doFilterRegionServerByName(Map<String, List<RegionInfo>> fullRsAndRMap) {</span> |
| <span class="source-line-no">1995</span><span id="line-1995"></span> |
| <span class="source-line-no">1996</span><span id="line-1996"> Map<String, List<RegionInfo>> filteredRsAndRMap = null;</span> |
| <span class="source-line-no">1997</span><span id="line-1997"></span> |
| <span class="source-line-no">1998</span><span id="line-1998"> if (this.targets != null && this.targets.length > 0) {</span> |
| <span class="source-line-no">1999</span><span id="line-1999"> filteredRsAndRMap = new HashMap<>();</span> |
| <span class="source-line-no">2000</span><span id="line-2000"> Pattern pattern = null;</span> |
| <span class="source-line-no">2001</span><span id="line-2001"> Matcher matcher = null;</span> |
| <span class="source-line-no">2002</span><span id="line-2002"> boolean regExpFound = false;</span> |
| <span class="source-line-no">2003</span><span id="line-2003"> for (String rsName : this.targets) {</span> |
| <span class="source-line-no">2004</span><span id="line-2004"> if (this.useRegExp) {</span> |
| <span class="source-line-no">2005</span><span id="line-2005"> regExpFound = false;</span> |
| <span class="source-line-no">2006</span><span id="line-2006"> pattern = Pattern.compile(rsName);</span> |
| <span class="source-line-no">2007</span><span id="line-2007"> for (Map.Entry<String, List<RegionInfo>> entry : fullRsAndRMap.entrySet()) {</span> |
| <span class="source-line-no">2008</span><span id="line-2008"> matcher = pattern.matcher(entry.getKey());</span> |
| <span class="source-line-no">2009</span><span id="line-2009"> if (matcher.matches()) {</span> |
| <span class="source-line-no">2010</span><span id="line-2010"> filteredRsAndRMap.put(entry.getKey(), entry.getValue());</span> |
| <span class="source-line-no">2011</span><span id="line-2011"> regExpFound = true;</span> |
| <span class="source-line-no">2012</span><span id="line-2012"> }</span> |
| <span class="source-line-no">2013</span><span id="line-2013"> }</span> |
| <span class="source-line-no">2014</span><span id="line-2014"> if (!regExpFound) {</span> |
| <span class="source-line-no">2015</span><span id="line-2015"> LOG.info("No RegionServerInfo found, regionServerPattern {}", rsName);</span> |
| <span class="source-line-no">2016</span><span id="line-2016"> }</span> |
| <span class="source-line-no">2017</span><span id="line-2017"> } else {</span> |
| <span class="source-line-no">2018</span><span id="line-2018"> if (fullRsAndRMap.containsKey(rsName)) {</span> |
| <span class="source-line-no">2019</span><span id="line-2019"> filteredRsAndRMap.put(rsName, fullRsAndRMap.get(rsName));</span> |
| <span class="source-line-no">2020</span><span id="line-2020"> } else {</span> |
| <span class="source-line-no">2021</span><span id="line-2021"> LOG.info("No RegionServerInfo found, regionServerName {}", rsName);</span> |
| <span class="source-line-no">2022</span><span id="line-2022"> }</span> |
| <span class="source-line-no">2023</span><span id="line-2023"> }</span> |
| <span class="source-line-no">2024</span><span id="line-2024"> }</span> |
| <span class="source-line-no">2025</span><span id="line-2025"> } else {</span> |
| <span class="source-line-no">2026</span><span id="line-2026"> filteredRsAndRMap = fullRsAndRMap;</span> |
| <span class="source-line-no">2027</span><span id="line-2027"> }</span> |
| <span class="source-line-no">2028</span><span id="line-2028"> return filteredRsAndRMap;</span> |
| <span class="source-line-no">2029</span><span id="line-2029"> }</span> |
| <span class="source-line-no">2030</span><span id="line-2030"> }</span> |
| <span class="source-line-no">2031</span><span id="line-2031"></span> |
| <span class="source-line-no">2032</span><span id="line-2032"> public static void main(String[] args) throws Exception {</span> |
| <span class="source-line-no">2033</span><span id="line-2033"> final Configuration conf = HBaseConfiguration.create();</span> |
| <span class="source-line-no">2034</span><span id="line-2034"></span> |
| <span class="source-line-no">2035</span><span id="line-2035"> int numThreads = conf.getInt("hbase.canary.threads.num", MAX_THREADS_NUM);</span> |
| <span class="source-line-no">2036</span><span id="line-2036"> LOG.info("Execution thread count={}", numThreads);</span> |
| <span class="source-line-no">2037</span><span id="line-2037"></span> |
| <span class="source-line-no">2038</span><span id="line-2038"> int exitCode;</span> |
| <span class="source-line-no">2039</span><span id="line-2039"> ExecutorService executor = new ScheduledThreadPoolExecutor(numThreads);</span> |
| <span class="source-line-no">2040</span><span id="line-2040"> try {</span> |
| <span class="source-line-no">2041</span><span id="line-2041"> exitCode = ToolRunner.run(conf, new CanaryTool(executor), args);</span> |
| <span class="source-line-no">2042</span><span id="line-2042"> } finally {</span> |
| <span class="source-line-no">2043</span><span id="line-2043"> executor.shutdown();</span> |
| <span class="source-line-no">2044</span><span id="line-2044"> }</span> |
| <span class="source-line-no">2045</span><span id="line-2045"> System.exit(exitCode);</span> |
| <span class="source-line-no">2046</span><span id="line-2046"> }</span> |
| <span class="source-line-no">2047</span><span id="line-2047">}</span> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </pre> |
| </div> |
| </main> |
| </body> |
| </html> |
