blob: fa33cb106d1cdd1f597a478ec2583d199f919775 [file] [log] [blame]
<!DOCTYPE HTML>
<html lang="en">
<head>
<!-- Generated by javadoc (17) -->
<title>Source code</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="source: package: org.apache.hadoop.hbase.util, class: HBaseFsck, class: WorkItemOverlapMerge">
<meta name="generator" content="javadoc/SourceToHTMLConverter">
<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style">
</head>
<body class="source-page">
<main role="main">
<div class="source-container">
<pre><span class="source-line-no">001</span><span id="line-1">/*</span>
<span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span>
<span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span>
<span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span>
<span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span>
<span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span>
<span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span>
<span class="source-line-no">009</span><span id="line-9"> *</span>
<span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="source-line-no">011</span><span id="line-11"> *</span>
<span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span>
<span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span>
<span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span>
<span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span>
<span class="source-line-no">017</span><span id="line-17"> */</span>
<span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.util;</span>
<span class="source-line-no">019</span><span id="line-19"></span>
<span class="source-line-no">020</span><span id="line-20">import java.io.Closeable;</span>
<span class="source-line-no">021</span><span id="line-21">import java.io.FileNotFoundException;</span>
<span class="source-line-no">022</span><span id="line-22">import java.io.IOException;</span>
<span class="source-line-no">023</span><span id="line-23">import java.io.InterruptedIOException;</span>
<span class="source-line-no">024</span><span id="line-24">import java.io.PrintWriter;</span>
<span class="source-line-no">025</span><span id="line-25">import java.io.StringWriter;</span>
<span class="source-line-no">026</span><span id="line-26">import java.net.InetAddress;</span>
<span class="source-line-no">027</span><span id="line-27">import java.net.URI;</span>
<span class="source-line-no">028</span><span id="line-28">import java.util.ArrayList;</span>
<span class="source-line-no">029</span><span id="line-29">import java.util.Collection;</span>
<span class="source-line-no">030</span><span id="line-30">import java.util.Collections;</span>
<span class="source-line-no">031</span><span id="line-31">import java.util.Comparator;</span>
<span class="source-line-no">032</span><span id="line-32">import java.util.EnumSet;</span>
<span class="source-line-no">033</span><span id="line-33">import java.util.HashMap;</span>
<span class="source-line-no">034</span><span id="line-34">import java.util.HashSet;</span>
<span class="source-line-no">035</span><span id="line-35">import java.util.Iterator;</span>
<span class="source-line-no">036</span><span id="line-36">import java.util.List;</span>
<span class="source-line-no">037</span><span id="line-37">import java.util.Locale;</span>
<span class="source-line-no">038</span><span id="line-38">import java.util.Map;</span>
<span class="source-line-no">039</span><span id="line-39">import java.util.Map.Entry;</span>
<span class="source-line-no">040</span><span id="line-40">import java.util.Objects;</span>
<span class="source-line-no">041</span><span id="line-41">import java.util.Optional;</span>
<span class="source-line-no">042</span><span id="line-42">import java.util.Set;</span>
<span class="source-line-no">043</span><span id="line-43">import java.util.SortedMap;</span>
<span class="source-line-no">044</span><span id="line-44">import java.util.TreeMap;</span>
<span class="source-line-no">045</span><span id="line-45">import java.util.Vector;</span>
<span class="source-line-no">046</span><span id="line-46">import java.util.concurrent.Callable;</span>
<span class="source-line-no">047</span><span id="line-47">import java.util.concurrent.ConcurrentSkipListMap;</span>
<span class="source-line-no">048</span><span id="line-48">import java.util.concurrent.ExecutionException;</span>
<span class="source-line-no">049</span><span id="line-49">import java.util.concurrent.ExecutorService;</span>
<span class="source-line-no">050</span><span id="line-50">import java.util.concurrent.Executors;</span>
<span class="source-line-no">051</span><span id="line-51">import java.util.concurrent.Future;</span>
<span class="source-line-no">052</span><span id="line-52">import java.util.concurrent.FutureTask;</span>
<span class="source-line-no">053</span><span id="line-53">import java.util.concurrent.ScheduledThreadPoolExecutor;</span>
<span class="source-line-no">054</span><span id="line-54">import java.util.concurrent.TimeUnit;</span>
<span class="source-line-no">055</span><span id="line-55">import java.util.concurrent.TimeoutException;</span>
<span class="source-line-no">056</span><span id="line-56">import java.util.concurrent.atomic.AtomicBoolean;</span>
<span class="source-line-no">057</span><span id="line-57">import java.util.concurrent.atomic.AtomicInteger;</span>
<span class="source-line-no">058</span><span id="line-58">import java.util.stream.Collectors;</span>
<span class="source-line-no">059</span><span id="line-59">import org.apache.commons.io.IOUtils;</span>
<span class="source-line-no">060</span><span id="line-60">import org.apache.commons.lang3.StringUtils;</span>
<span class="source-line-no">061</span><span id="line-61">import org.apache.hadoop.conf.Configuration;</span>
<span class="source-line-no">062</span><span id="line-62">import org.apache.hadoop.conf.Configured;</span>
<span class="source-line-no">063</span><span id="line-63">import org.apache.hadoop.fs.FSDataOutputStream;</span>
<span class="source-line-no">064</span><span id="line-64">import org.apache.hadoop.fs.FileStatus;</span>
<span class="source-line-no">065</span><span id="line-65">import org.apache.hadoop.fs.FileSystem;</span>
<span class="source-line-no">066</span><span id="line-66">import org.apache.hadoop.fs.Path;</span>
<span class="source-line-no">067</span><span id="line-67">import org.apache.hadoop.fs.permission.FsAction;</span>
<span class="source-line-no">068</span><span id="line-68">import org.apache.hadoop.fs.permission.FsPermission;</span>
<span class="source-line-no">069</span><span id="line-69">import org.apache.hadoop.hbase.Abortable;</span>
<span class="source-line-no">070</span><span id="line-70">import org.apache.hadoop.hbase.CatalogFamilyFormat;</span>
<span class="source-line-no">071</span><span id="line-71">import org.apache.hadoop.hbase.Cell;</span>
<span class="source-line-no">072</span><span id="line-72">import org.apache.hadoop.hbase.CellUtil;</span>
<span class="source-line-no">073</span><span id="line-73">import org.apache.hadoop.hbase.ClientMetaTableAccessor;</span>
<span class="source-line-no">074</span><span id="line-74">import org.apache.hadoop.hbase.ClusterMetrics;</span>
<span class="source-line-no">075</span><span id="line-75">import org.apache.hadoop.hbase.ClusterMetrics.Option;</span>
<span class="source-line-no">076</span><span id="line-76">import org.apache.hadoop.hbase.ExtendedCell;</span>
<span class="source-line-no">077</span><span id="line-77">import org.apache.hadoop.hbase.HBaseConfiguration;</span>
<span class="source-line-no">078</span><span id="line-78">import org.apache.hadoop.hbase.HBaseInterfaceAudience;</span>
<span class="source-line-no">079</span><span id="line-79">import org.apache.hadoop.hbase.HConstants;</span>
<span class="source-line-no">080</span><span id="line-80">import org.apache.hadoop.hbase.HRegionLocation;</span>
<span class="source-line-no">081</span><span id="line-81">import org.apache.hadoop.hbase.KeyValue;</span>
<span class="source-line-no">082</span><span id="line-82">import org.apache.hadoop.hbase.MasterNotRunningException;</span>
<span class="source-line-no">083</span><span id="line-83">import org.apache.hadoop.hbase.MetaTableAccessor;</span>
<span class="source-line-no">084</span><span id="line-84">import org.apache.hadoop.hbase.RegionLocations;</span>
<span class="source-line-no">085</span><span id="line-85">import org.apache.hadoop.hbase.ServerName;</span>
<span class="source-line-no">086</span><span id="line-86">import org.apache.hadoop.hbase.TableName;</span>
<span class="source-line-no">087</span><span id="line-87">import org.apache.hadoop.hbase.TableNotFoundException;</span>
<span class="source-line-no">088</span><span id="line-88">import org.apache.hadoop.hbase.ZooKeeperConnectionException;</span>
<span class="source-line-no">089</span><span id="line-89">import org.apache.hadoop.hbase.client.Admin;</span>
<span class="source-line-no">090</span><span id="line-90">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span>
<span class="source-line-no">091</span><span id="line-91">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span>
<span class="source-line-no">092</span><span id="line-92">import org.apache.hadoop.hbase.client.Connection;</span>
<span class="source-line-no">093</span><span id="line-93">import org.apache.hadoop.hbase.client.ConnectionFactory;</span>
<span class="source-line-no">094</span><span id="line-94">import org.apache.hadoop.hbase.client.Delete;</span>
<span class="source-line-no">095</span><span id="line-95">import org.apache.hadoop.hbase.client.Get;</span>
<span class="source-line-no">096</span><span id="line-96">import org.apache.hadoop.hbase.client.Put;</span>
<span class="source-line-no">097</span><span id="line-97">import org.apache.hadoop.hbase.client.RegionInfo;</span>
<span class="source-line-no">098</span><span id="line-98">import org.apache.hadoop.hbase.client.RegionInfoBuilder;</span>
<span class="source-line-no">099</span><span id="line-99">import org.apache.hadoop.hbase.client.RegionLocator;</span>
<span class="source-line-no">100</span><span id="line-100">import org.apache.hadoop.hbase.client.RegionReplicaUtil;</span>
<span class="source-line-no">101</span><span id="line-101">import org.apache.hadoop.hbase.client.Result;</span>
<span class="source-line-no">102</span><span id="line-102">import org.apache.hadoop.hbase.client.ResultScanner;</span>
<span class="source-line-no">103</span><span id="line-103">import org.apache.hadoop.hbase.client.RowMutations;</span>
<span class="source-line-no">104</span><span id="line-104">import org.apache.hadoop.hbase.client.Scan;</span>
<span class="source-line-no">105</span><span id="line-105">import org.apache.hadoop.hbase.client.Table;</span>
<span class="source-line-no">106</span><span id="line-106">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
<span class="source-line-no">107</span><span id="line-107">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span>
<span class="source-line-no">108</span><span id="line-108">import org.apache.hadoop.hbase.client.TableState;</span>
<span class="source-line-no">109</span><span id="line-109">import org.apache.hadoop.hbase.io.FileLink;</span>
<span class="source-line-no">110</span><span id="line-110">import org.apache.hadoop.hbase.io.HFileLink;</span>
<span class="source-line-no">111</span><span id="line-111">import org.apache.hadoop.hbase.io.hfile.CacheConfig;</span>
<span class="source-line-no">112</span><span id="line-112">import org.apache.hadoop.hbase.io.hfile.HFile;</span>
<span class="source-line-no">113</span><span id="line-113">import org.apache.hadoop.hbase.master.RegionState;</span>
<span class="source-line-no">114</span><span id="line-114">import org.apache.hadoop.hbase.regionserver.HRegion;</span>
<span class="source-line-no">115</span><span id="line-115">import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;</span>
<span class="source-line-no">116</span><span id="line-116">import org.apache.hadoop.hbase.regionserver.StoreFileInfo;</span>
<span class="source-line-no">117</span><span id="line-117">import org.apache.hadoop.hbase.replication.ReplicationException;</span>
<span class="source-line-no">118</span><span id="line-118">import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;</span>
<span class="source-line-no">119</span><span id="line-119">import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;</span>
<span class="source-line-no">120</span><span id="line-120">import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;</span>
<span class="source-line-no">121</span><span id="line-121">import org.apache.hadoop.hbase.security.UserProvider;</span>
<span class="source-line-no">122</span><span id="line-122">import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;</span>
<span class="source-line-no">123</span><span id="line-123">import org.apache.hadoop.hbase.util.HbckErrorReporter.ERROR_CODE;</span>
<span class="source-line-no">124</span><span id="line-124">import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;</span>
<span class="source-line-no">125</span><span id="line-125">import org.apache.hadoop.hbase.util.hbck.ReplicationChecker;</span>
<span class="source-line-no">126</span><span id="line-126">import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler;</span>
<span class="source-line-no">127</span><span id="line-127">import org.apache.hadoop.hbase.wal.WALSplitUtil;</span>
<span class="source-line-no">128</span><span id="line-128">import org.apache.hadoop.hbase.zookeeper.ZKUtil;</span>
<span class="source-line-no">129</span><span id="line-129">import org.apache.hadoop.hbase.zookeeper.ZKWatcher;</span>
<span class="source-line-no">130</span><span id="line-130">import org.apache.hadoop.hbase.zookeeper.ZNodePaths;</span>
<span class="source-line-no">131</span><span id="line-131">import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;</span>
<span class="source-line-no">132</span><span id="line-132">import org.apache.hadoop.ipc.RemoteException;</span>
<span class="source-line-no">133</span><span id="line-133">import org.apache.hadoop.security.AccessControlException;</span>
<span class="source-line-no">134</span><span id="line-134">import org.apache.hadoop.security.UserGroupInformation;</span>
<span class="source-line-no">135</span><span id="line-135">import org.apache.hadoop.util.ReflectionUtils;</span>
<span class="source-line-no">136</span><span id="line-136">import org.apache.hadoop.util.Tool;</span>
<span class="source-line-no">137</span><span id="line-137">import org.apache.hadoop.util.ToolRunner;</span>
<span class="source-line-no">138</span><span id="line-138">import org.apache.yetus.audience.InterfaceAudience;</span>
<span class="source-line-no">139</span><span id="line-139">import org.apache.yetus.audience.InterfaceStability;</span>
<span class="source-line-no">140</span><span id="line-140">import org.apache.zookeeper.KeeperException;</span>
<span class="source-line-no">141</span><span id="line-141">import org.slf4j.Logger;</span>
<span class="source-line-no">142</span><span id="line-142">import org.slf4j.LoggerFactory;</span>
<span class="source-line-no">143</span><span id="line-143"></span>
<span class="source-line-no">144</span><span id="line-144">import org.apache.hbase.thirdparty.com.google.common.base.Joiner;</span>
<span class="source-line-no">145</span><span id="line-145">import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;</span>
<span class="source-line-no">146</span><span id="line-146">import org.apache.hbase.thirdparty.com.google.common.collect.Lists;</span>
<span class="source-line-no">147</span><span id="line-147">import org.apache.hbase.thirdparty.com.google.common.collect.Sets;</span>
<span class="source-line-no">148</span><span id="line-148">import org.apache.hbase.thirdparty.com.google.common.io.Closeables;</span>
<span class="source-line-no">149</span><span id="line-149">import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;</span>
<span class="source-line-no">150</span><span id="line-150"></span>
<span class="source-line-no">151</span><span id="line-151">/**</span>
<span class="source-line-no">152</span><span id="line-152"> * HBaseFsck (hbck) is a tool for checking and repairing region consistency and table integrity</span>
<span class="source-line-no">153</span><span id="line-153"> * problems in a corrupted HBase. This tool was written for hbase-1.x. It does not work with</span>
<span class="source-line-no">154</span><span id="line-154"> * hbase-2.x; it can read state but is not allowed to change state; i.e. effect 'repair'. Even</span>
<span class="source-line-no">155</span><span id="line-155"> * though it can 'read' state, given how so much has changed in how hbase1 and hbase2 operate, it</span>
<span class="source-line-no">156</span><span id="line-156"> * will often misread. See hbck2 (HBASE-19121) for a hbck tool for hbase2. This class is deprecated.</span>
<span class="source-line-no">157</span><span id="line-157"> * &lt;p&gt;</span>
<span class="source-line-no">158</span><span id="line-158"> * Region consistency checks verify that hbase:meta, region deployment on region servers and the</span>
<span class="source-line-no">159</span><span id="line-159"> * state of data in HDFS (.regioninfo files) all are in accordance.</span>
<span class="source-line-no">160</span><span id="line-160"> * &lt;p&gt;</span>
<span class="source-line-no">161</span><span id="line-161"> * Table integrity checks verify that all possible row keys resolve to exactly one region of a</span>
<span class="source-line-no">162</span><span id="line-162"> * table. This means there are no individual degenerate or backwards regions; no holes between</span>
<span class="source-line-no">163</span><span id="line-163"> * regions; and that there are no overlapping regions.</span>
<span class="source-line-no">164</span><span id="line-164"> * &lt;p&gt;</span>
<span class="source-line-no">165</span><span id="line-165"> * The general repair strategy works in two phases:</span>
<span class="source-line-no">166</span><span id="line-166"> * &lt;ol&gt;</span>
<span class="source-line-no">167</span><span id="line-167"> * &lt;li&gt;Repair Table Integrity on HDFS. (merge or fabricate regions)</span>
<span class="source-line-no">168</span><span id="line-168"> * &lt;li&gt;Repair Region Consistency with hbase:meta and assignments</span>
<span class="source-line-no">169</span><span id="line-169"> * &lt;/ol&gt;</span>
<span class="source-line-no">170</span><span id="line-170"> * &lt;p&gt;</span>
<span class="source-line-no">171</span><span id="line-171"> * For table integrity repairs, the tables' region directories are scanned for .regioninfo files.</span>
<span class="source-line-no">172</span><span id="line-172"> * Each table's integrity is then verified. If there are any orphan regions (regions with no</span>
<span class="source-line-no">173</span><span id="line-173"> * .regioninfo files) or holes, new regions are fabricated. Backwards regions are sidelined as well</span>
<span class="source-line-no">174</span><span id="line-174"> * as empty degenerate (endkey==startkey) regions. If there are any overlapping regions, a new</span>
<span class="source-line-no">175</span><span id="line-175"> * region is created and all data is merged into the new region.</span>
<span class="source-line-no">176</span><span id="line-176"> * &lt;p&gt;</span>
<span class="source-line-no">177</span><span id="line-177"> * Table integrity repairs deal solely with HDFS and could potentially be done offline -- the hbase</span>
<span class="source-line-no">178</span><span id="line-178"> * region servers or master do not need to be running. This phase can eventually be used to</span>
<span class="source-line-no">179</span><span id="line-179"> * completely reconstruct the hbase:meta table in an offline fashion.</span>
<span class="source-line-no">180</span><span id="line-180"> * &lt;p&gt;</span>
<span class="source-line-no">181</span><span id="line-181"> * Region consistency requires three conditions -- 1) valid .regioninfo file present in an HDFS</span>
<span class="source-line-no">182</span><span id="line-182"> * region dir, 2) valid row with .regioninfo data in META, and 3) a region is deployed only at the</span>
<span class="source-line-no">183</span><span id="line-183"> * regionserver that was assigned to with proper state in the master.</span>
<span class="source-line-no">184</span><span id="line-184"> * &lt;p&gt;</span>
<span class="source-line-no">185</span><span id="line-185"> * Region consistency repairs require hbase to be online so that hbck can contact the HBase master</span>
<span class="source-line-no">186</span><span id="line-186"> * and region servers. The hbck#connect() method must first be called successfully. Much of the</span>
<span class="source-line-no">187</span><span id="line-187"> * region consistency information is transient and less risky to repair.</span>
<span class="source-line-no">188</span><span id="line-188"> * &lt;p&gt;</span>
<span class="source-line-no">189</span><span id="line-189"> * If hbck is run from the command line, there are a handful of arguments that can be used to limit</span>
<span class="source-line-no">190</span><span id="line-190"> * the kinds of repairs hbck will do. See the code in {@link #printUsageAndExit()} for more details.</span>
<span class="source-line-no">191</span><span id="line-191"> * @deprecated For removal in hbase-4.0.0. Use HBCK2 instead.</span>
<span class="source-line-no">192</span><span id="line-192"> */</span>
<span class="source-line-no">193</span><span id="line-193">@Deprecated</span>
<span class="source-line-no">194</span><span id="line-194">@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)</span>
<span class="source-line-no">195</span><span id="line-195">@InterfaceStability.Evolving</span>
<span class="source-line-no">196</span><span id="line-196">public class HBaseFsck extends Configured implements Closeable {</span>
<span class="source-line-no">197</span><span id="line-197"> public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute</span>
<span class="source-line-no">198</span><span id="line-198"> public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;</span>
<span class="source-line-no">199</span><span id="line-199"> private static final int MAX_NUM_THREADS = 50; // #threads to contact regions</span>
<span class="source-line-no">200</span><span id="line-200"> private static boolean rsSupportsOffline = true;</span>
<span class="source-line-no">201</span><span id="line-201"> private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;</span>
<span class="source-line-no">202</span><span id="line-202"> private static final int DEFAULT_MAX_MERGE = 5;</span>
<span class="source-line-no">203</span><span id="line-203"></span>
<span class="source-line-no">204</span><span id="line-204"> /**</span>
<span class="source-line-no">205</span><span id="line-205"> * Here is where hbase-1.x used to default the lock for hbck1. It puts in place a lock when it</span>
<span class="source-line-no">206</span><span id="line-206"> * goes to write/make changes.</span>
<span class="source-line-no">207</span><span id="line-207"> */</span>
<span class="source-line-no">208</span><span id="line-208"> @InterfaceAudience.Private</span>
<span class="source-line-no">209</span><span id="line-209"> public static final String HBCK_LOCK_FILE = "hbase-hbck.lock";</span>
<span class="source-line-no">210</span><span id="line-210"> private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5;</span>
<span class="source-line-no">211</span><span id="line-211"> private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds</span>
<span class="source-line-no">212</span><span id="line-212"> private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds</span>
<span class="source-line-no">213</span><span id="line-213"> // We have to set the timeout value &gt; HdfsConstants.LEASE_SOFTLIMIT_PERIOD.</span>
<span class="source-line-no">214</span><span id="line-214"> // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for</span>
<span class="source-line-no">215</span><span id="line-215"> // AlreadyBeingCreatedException which is implies timeout on this operations up to</span>
<span class="source-line-no">216</span><span id="line-216"> // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).</span>
<span class="source-line-no">217</span><span id="line-217"> private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds</span>
<span class="source-line-no">218</span><span id="line-218"> private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5;</span>
<span class="source-line-no">219</span><span id="line-219"> private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds</span>
<span class="source-line-no">220</span><span id="line-220"> private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds</span>
<span class="source-line-no">221</span><span id="line-221"></span>
<span class="source-line-no">222</span><span id="line-222"> /**********************</span>
<span class="source-line-no">223</span><span id="line-223"> * Internal resources</span>
<span class="source-line-no">224</span><span id="line-224"> **********************/</span>
<span class="source-line-no">225</span><span id="line-225"> private static final Logger LOG = LoggerFactory.getLogger(HBaseFsck.class.getName());</span>
<span class="source-line-no">226</span><span id="line-226"> private ClusterMetrics status;</span>
<span class="source-line-no">227</span><span id="line-227"> private Connection connection;</span>
<span class="source-line-no">228</span><span id="line-228"> private Admin admin;</span>
<span class="source-line-no">229</span><span id="line-229"> private Table meta;</span>
<span class="source-line-no">230</span><span id="line-230"> // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions</span>
<span class="source-line-no">231</span><span id="line-231"> protected ExecutorService executor;</span>
<span class="source-line-no">232</span><span id="line-232"> private long startMillis = EnvironmentEdgeManager.currentTime();</span>
<span class="source-line-no">233</span><span id="line-233"> private HFileCorruptionChecker hfcc;</span>
<span class="source-line-no">234</span><span id="line-234"> private int retcode = 0;</span>
<span class="source-line-no">235</span><span id="line-235"> private Path HBCK_LOCK_PATH;</span>
<span class="source-line-no">236</span><span id="line-236"> private FSDataOutputStream hbckOutFd;</span>
<span class="source-line-no">237</span><span id="line-237"> // This lock is to prevent cleanup of balancer resources twice between</span>
<span class="source-line-no">238</span><span id="line-238"> // ShutdownHook and the main code. We cleanup only if the connect() is</span>
<span class="source-line-no">239</span><span id="line-239"> // successful</span>
<span class="source-line-no">240</span><span id="line-240"> private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);</span>
<span class="source-line-no">241</span><span id="line-241"></span>
<span class="source-line-no">242</span><span id="line-242"> // Unsupported options in HBase 2.0+</span>
<span class="source-line-no">243</span><span id="line-243"> private static final Set&lt;String&gt; unsupportedOptionsInV2 = Sets.newHashSet("-fix",</span>
<span class="source-line-no">244</span><span id="line-244"> "-fixAssignments", "-fixMeta", "-fixHdfsHoles", "-fixHdfsOrphans", "-fixTableOrphans",</span>
<span class="source-line-no">245</span><span id="line-245"> "-fixHdfsOverlaps", "-sidelineBigOverlaps", "-fixSplitParents", "-removeParents",</span>
<span class="source-line-no">246</span><span id="line-246"> "-fixEmptyMetaCells", "-repair", "-repairHoles", "-maxOverlapsToSideline", "-maxMerge");</span>
<span class="source-line-no">247</span><span id="line-247"></span>
<span class="source-line-no">248</span><span id="line-248"> /***********</span>
<span class="source-line-no">249</span><span id="line-249"> * Options</span>
<span class="source-line-no">250</span><span id="line-250"> ***********/</span>
<span class="source-line-no">251</span><span id="line-251"> private static boolean details = false; // do we display the full report</span>
<span class="source-line-no">252</span><span id="line-252"> private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older</span>
<span class="source-line-no">253</span><span id="line-253"> private static boolean forceExclusive = false; // only this hbck can modify HBase</span>
<span class="source-line-no">254</span><span id="line-254"> private boolean fixAssignments = false; // fix assignment errors?</span>
<span class="source-line-no">255</span><span id="line-255"> private boolean fixMeta = false; // fix meta errors?</span>
<span class="source-line-no">256</span><span id="line-256"> private boolean checkHdfs = true; // load and check fs consistency?</span>
<span class="source-line-no">257</span><span id="line-257"> private boolean fixHdfsHoles = false; // fix fs holes?</span>
<span class="source-line-no">258</span><span id="line-258"> private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)</span>
<span class="source-line-no">259</span><span id="line-259"> private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)</span>
<span class="source-line-no">260</span><span id="line-260"> private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)</span>
<span class="source-line-no">261</span><span id="line-261"> private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs</span>
<span class="source-line-no">262</span><span id="line-262"> private boolean fixSplitParents = false; // fix lingering split parents</span>
<span class="source-line-no">263</span><span id="line-263"> private boolean removeParents = false; // remove split parents</span>
<span class="source-line-no">264</span><span id="line-264"> private boolean fixReferenceFiles = false; // fix lingering reference store file</span>
<span class="source-line-no">265</span><span id="line-265"> private boolean fixHFileLinks = false; // fix lingering HFileLinks</span>
<span class="source-line-no">266</span><span id="line-266"> private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows</span>
<span class="source-line-no">267</span><span id="line-267"> private boolean fixReplication = false; // fix undeleted replication queues for removed peer</span>
<span class="source-line-no">268</span><span id="line-268"> private boolean cleanReplicationBarrier = false; // clean replication barriers of a table</span>
<span class="source-line-no">269</span><span id="line-269"> private boolean fixAny = false; // Set to true if any of the fix is required.</span>
<span class="source-line-no">270</span><span id="line-270"></span>
<span class="source-line-no">271</span><span id="line-271"> // limit checking/fixes to listed tables, if empty attempt to check/fix all</span>
<span class="source-line-no">272</span><span id="line-272"> // hbase:meta are always checked</span>
<span class="source-line-no">273</span><span id="line-273"> private Set&lt;TableName&gt; tablesIncluded = new HashSet&lt;&gt;();</span>
<span class="source-line-no">274</span><span id="line-274"> private TableName cleanReplicationBarrierTable;</span>
<span class="source-line-no">275</span><span id="line-275"> private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge</span>
<span class="source-line-no">276</span><span id="line-276"> // maximum number of overlapping regions to sideline</span>
<span class="source-line-no">277</span><span id="line-277"> private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE;</span>
<span class="source-line-no">278</span><span id="line-278"> private boolean sidelineBigOverlaps = false; // sideline overlaps with &gt;maxMerge regions</span>
<span class="source-line-no">279</span><span id="line-279"> private Path sidelineDir = null;</span>
<span class="source-line-no">280</span><span id="line-280"></span>
<span class="source-line-no">281</span><span id="line-281"> private boolean rerun = false; // if we tried to fix something, rerun hbck</span>
<span class="source-line-no">282</span><span id="line-282"> private static boolean summary = false; // if we want to print less output</span>
<span class="source-line-no">283</span><span id="line-283"> private boolean checkMetaOnly = false;</span>
<span class="source-line-no">284</span><span id="line-284"> private boolean checkRegionBoundaries = false;</span>
<span class="source-line-no">285</span><span id="line-285"> private boolean ignorePreCheckPermission = false; // if pre-check permission</span>
<span class="source-line-no">286</span><span id="line-286"></span>
<span class="source-line-no">287</span><span id="line-287"> /*********</span>
<span class="source-line-no">288</span><span id="line-288"> * State</span>
<span class="source-line-no">289</span><span id="line-289"> *********/</span>
<span class="source-line-no">290</span><span id="line-290"> final private HbckErrorReporter errors;</span>
<span class="source-line-no">291</span><span id="line-291"> int fixes = 0;</span>
<span class="source-line-no">292</span><span id="line-292"></span>
<span class="source-line-no">293</span><span id="line-293"> /**</span>
<span class="source-line-no">294</span><span id="line-294"> * This map contains the state of all hbck items. It maps from encoded region name to</span>
<span class="source-line-no">295</span><span id="line-295"> * HbckRegionInfo structure. The information contained in HbckRegionInfo is used to detect and</span>
<span class="source-line-no">296</span><span id="line-296"> * correct consistency (hdfs/meta/deployment) problems.</span>
<span class="source-line-no">297</span><span id="line-297"> */</span>
<span class="source-line-no">298</span><span id="line-298"> private TreeMap&lt;String, HbckRegionInfo&gt; regionInfoMap = new TreeMap&lt;&gt;();</span>
<span class="source-line-no">299</span><span id="line-299"> // Empty regioninfo qualifiers in hbase:meta</span>
<span class="source-line-no">300</span><span id="line-300"> private Set&lt;Result&gt; emptyRegionInfoQualifiers = new HashSet&lt;&gt;();</span>
<span class="source-line-no">301</span><span id="line-301"></span>
<span class="source-line-no">302</span><span id="line-302"> /**</span>
<span class="source-line-no">303</span><span id="line-303"> * This map from Tablename -&gt; TableInfo contains the structures necessary to detect table</span>
<span class="source-line-no">304</span><span id="line-304"> * consistency problems (holes, dupes, overlaps). It is sorted to prevent dupes. If tablesIncluded</span>
<span class="source-line-no">305</span><span id="line-305"> * is empty, this map contains all tables. Otherwise, it contains only meta tables and tables in</span>
<span class="source-line-no">306</span><span id="line-306"> * tablesIncluded, unless checkMetaOnly is specified, in which case, it contains only the meta</span>
<span class="source-line-no">307</span><span id="line-307"> * table</span>
<span class="source-line-no">308</span><span id="line-308"> */</span>
<span class="source-line-no">309</span><span id="line-309"> private SortedMap&lt;TableName, HbckTableInfo&gt; tablesInfo = new ConcurrentSkipListMap&lt;&gt;();</span>
<span class="source-line-no">310</span><span id="line-310"></span>
<span class="source-line-no">311</span><span id="line-311"> /**</span>
<span class="source-line-no">312</span><span id="line-312"> * When initially looking at HDFS, we attempt to find any orphaned data.</span>
<span class="source-line-no">313</span><span id="line-313"> */</span>
<span class="source-line-no">314</span><span id="line-314"> private List&lt;HbckRegionInfo&gt; orphanHdfsDirs = Collections.synchronizedList(new ArrayList&lt;&gt;());</span>
<span class="source-line-no">315</span><span id="line-315"></span>
<span class="source-line-no">316</span><span id="line-316"> private Map&lt;TableName, Set&lt;String&gt;&gt; orphanTableDirs = new HashMap&lt;&gt;();</span>
<span class="source-line-no">317</span><span id="line-317"> private Map&lt;TableName, TableState&gt; tableStates = new HashMap&lt;&gt;();</span>
<span class="source-line-no">318</span><span id="line-318"> private final RetryCounterFactory lockFileRetryCounterFactory;</span>
<span class="source-line-no">319</span><span id="line-319"> private final RetryCounterFactory createZNodeRetryCounterFactory;</span>
<span class="source-line-no">320</span><span id="line-320"></span>
<span class="source-line-no">321</span><span id="line-321"> private Map&lt;TableName, Set&lt;String&gt;&gt; skippedRegions = new HashMap&lt;&gt;();</span>
<span class="source-line-no">322</span><span id="line-322"></span>
<span class="source-line-no">323</span><span id="line-323"> private ZKWatcher zkw = null;</span>
<span class="source-line-no">324</span><span id="line-324"> private String hbckEphemeralNodePath = null;</span>
<span class="source-line-no">325</span><span id="line-325"> private boolean hbckZodeCreated = false;</span>
<span class="source-line-no">326</span><span id="line-326"></span>
<span class="source-line-no">327</span><span id="line-327"> /**</span>
<span class="source-line-no">328</span><span id="line-328"> * Constructor</span>
<span class="source-line-no">329</span><span id="line-329"> * @param conf Configuration object</span>
<span class="source-line-no">330</span><span id="line-330"> * @throws MasterNotRunningException if the master is not running</span>
<span class="source-line-no">331</span><span id="line-331"> * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper</span>
<span class="source-line-no">332</span><span id="line-332"> */</span>
<span class="source-line-no">333</span><span id="line-333"> public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException {</span>
<span class="source-line-no">334</span><span id="line-334"> this(conf, createThreadPool(conf));</span>
<span class="source-line-no">335</span><span id="line-335"> }</span>
<span class="source-line-no">336</span><span id="line-336"></span>
<span class="source-line-no">337</span><span id="line-337"> private static ExecutorService createThreadPool(Configuration conf) {</span>
<span class="source-line-no">338</span><span id="line-338"> int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);</span>
<span class="source-line-no">339</span><span id="line-339"> return new ScheduledThreadPoolExecutor(numThreads,</span>
<span class="source-line-no">340</span><span id="line-340"> new ThreadFactoryBuilder().setNameFormat("hbasefsck-pool-%d").setDaemon(true)</span>
<span class="source-line-no">341</span><span id="line-341"> .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());</span>
<span class="source-line-no">342</span><span id="line-342"> }</span>
<span class="source-line-no">343</span><span id="line-343"></span>
<span class="source-line-no">344</span><span id="line-344"> /**</span>
<span class="source-line-no">345</span><span id="line-345"> * Constructor Configuration object if the master is not running if unable to connect to ZooKeeper</span>
<span class="source-line-no">346</span><span id="line-346"> */</span>
<span class="source-line-no">347</span><span id="line-347"> public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,</span>
<span class="source-line-no">348</span><span id="line-348"> ZooKeeperConnectionException, IOException, ClassNotFoundException {</span>
<span class="source-line-no">349</span><span id="line-349"> super(conf);</span>
<span class="source-line-no">350</span><span id="line-350"> errors = getErrorReporter(getConf());</span>
<span class="source-line-no">351</span><span id="line-351"> this.executor = exec;</span>
<span class="source-line-no">352</span><span id="line-352"> lockFileRetryCounterFactory = createLockRetryCounterFactory(getConf());</span>
<span class="source-line-no">353</span><span id="line-353"> createZNodeRetryCounterFactory = createZnodeRetryCounterFactory(getConf());</span>
<span class="source-line-no">354</span><span id="line-354"> zkw = createZooKeeperWatcher();</span>
<span class="source-line-no">355</span><span id="line-355"> }</span>
<span class="source-line-no">356</span><span id="line-356"></span>
<span class="source-line-no">357</span><span id="line-357"> /** Returns A retry counter factory configured for retrying lock file creation. */</span>
<span class="source-line-no">358</span><span id="line-358"> public static RetryCounterFactory createLockRetryCounterFactory(Configuration conf) {</span>
<span class="source-line-no">359</span><span id="line-359"> return new RetryCounterFactory(</span>
<span class="source-line-no">360</span><span id="line-360"> conf.getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS),</span>
<span class="source-line-no">361</span><span id="line-361"> conf.getInt("hbase.hbck.lockfile.attempt.sleep.interval",</span>
<span class="source-line-no">362</span><span id="line-362"> DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),</span>
<span class="source-line-no">363</span><span id="line-363"> conf.getInt("hbase.hbck.lockfile.attempt.maxsleeptime",</span>
<span class="source-line-no">364</span><span id="line-364"> DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));</span>
<span class="source-line-no">365</span><span id="line-365"> }</span>
<span class="source-line-no">366</span><span id="line-366"></span>
<span class="source-line-no">367</span><span id="line-367"> /** Returns A retry counter factory configured for retrying znode creation. */</span>
<span class="source-line-no">368</span><span id="line-368"> private static RetryCounterFactory createZnodeRetryCounterFactory(Configuration conf) {</span>
<span class="source-line-no">369</span><span id="line-369"> return new RetryCounterFactory(</span>
<span class="source-line-no">370</span><span id="line-370"> conf.getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS),</span>
<span class="source-line-no">371</span><span id="line-371"> conf.getInt("hbase.hbck.createznode.attempt.sleep.interval",</span>
<span class="source-line-no">372</span><span id="line-372"> DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL),</span>
<span class="source-line-no">373</span><span id="line-373"> conf.getInt("hbase.hbck.createznode.attempt.maxsleeptime",</span>
<span class="source-line-no">374</span><span id="line-374"> DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME));</span>
<span class="source-line-no">375</span><span id="line-375"> }</span>
<span class="source-line-no">376</span><span id="line-376"></span>
<span class="source-line-no">377</span><span id="line-377"> /** Returns Return the tmp dir this tool writes too. */</span>
<span class="source-line-no">378</span><span id="line-378"> @InterfaceAudience.Private</span>
<span class="source-line-no">379</span><span id="line-379"> public static Path getTmpDir(Configuration conf) throws IOException {</span>
<span class="source-line-no">380</span><span id="line-380"> return new Path(CommonFSUtils.getRootDir(conf), HConstants.HBASE_TEMP_DIRECTORY);</span>
<span class="source-line-no">381</span><span id="line-381"> }</span>
<span class="source-line-no">382</span><span id="line-382"></span>
<span class="source-line-no">383</span><span id="line-383"> private static class FileLockCallable implements Callable&lt;FSDataOutputStream&gt; {</span>
<span class="source-line-no">384</span><span id="line-384"> RetryCounter retryCounter;</span>
<span class="source-line-no">385</span><span id="line-385"> private final Configuration conf;</span>
<span class="source-line-no">386</span><span id="line-386"> private Path hbckLockPath = null;</span>
<span class="source-line-no">387</span><span id="line-387"></span>
<span class="source-line-no">388</span><span id="line-388"> public FileLockCallable(Configuration conf, RetryCounter retryCounter) {</span>
<span class="source-line-no">389</span><span id="line-389"> this.retryCounter = retryCounter;</span>
<span class="source-line-no">390</span><span id="line-390"> this.conf = conf;</span>
<span class="source-line-no">391</span><span id="line-391"> }</span>
<span class="source-line-no">392</span><span id="line-392"></span>
<span class="source-line-no">393</span><span id="line-393"> /** Returns Will be &lt;code&gt;null&lt;/code&gt; unless you call {@link #call()} */</span>
<span class="source-line-no">394</span><span id="line-394"> Path getHbckLockPath() {</span>
<span class="source-line-no">395</span><span id="line-395"> return this.hbckLockPath;</span>
<span class="source-line-no">396</span><span id="line-396"> }</span>
<span class="source-line-no">397</span><span id="line-397"></span>
<span class="source-line-no">398</span><span id="line-398"> @Override</span>
<span class="source-line-no">399</span><span id="line-399"> public FSDataOutputStream call() throws IOException {</span>
<span class="source-line-no">400</span><span id="line-400"> try {</span>
<span class="source-line-no">401</span><span id="line-401"> FileSystem fs = CommonFSUtils.getCurrentFileSystem(this.conf);</span>
<span class="source-line-no">402</span><span id="line-402"> FsPermission defaultPerms =</span>
<span class="source-line-no">403</span><span id="line-403"> CommonFSUtils.getFilePermissions(fs, this.conf, HConstants.DATA_FILE_UMASK_KEY);</span>
<span class="source-line-no">404</span><span id="line-404"> Path tmpDir = getTmpDir(conf);</span>
<span class="source-line-no">405</span><span id="line-405"> this.hbckLockPath = new Path(tmpDir, HBCK_LOCK_FILE);</span>
<span class="source-line-no">406</span><span id="line-406"> fs.mkdirs(tmpDir);</span>
<span class="source-line-no">407</span><span id="line-407"> final FSDataOutputStream out = createFileWithRetries(fs, this.hbckLockPath, defaultPerms);</span>
<span class="source-line-no">408</span><span id="line-408"> out.writeBytes(InetAddress.getLocalHost().toString());</span>
<span class="source-line-no">409</span><span id="line-409"> // Add a note into the file we write on why hbase2 is writing out an hbck1 lock file.</span>
<span class="source-line-no">410</span><span id="line-410"> out.writeBytes(" Written by an hbase-2.x Master to block an "</span>
<span class="source-line-no">411</span><span id="line-411"> + "attempt by an hbase-1.x HBCK tool making modification to state. "</span>
<span class="source-line-no">412</span><span id="line-412"> + "See 'HBCK must match HBase server version' in the hbase refguide.");</span>
<span class="source-line-no">413</span><span id="line-413"> out.flush();</span>
<span class="source-line-no">414</span><span id="line-414"> return out;</span>
<span class="source-line-no">415</span><span id="line-415"> } catch (RemoteException e) {</span>
<span class="source-line-no">416</span><span id="line-416"> if (AlreadyBeingCreatedException.class.getName().equals(e.getClassName())) {</span>
<span class="source-line-no">417</span><span id="line-417"> return null;</span>
<span class="source-line-no">418</span><span id="line-418"> } else {</span>
<span class="source-line-no">419</span><span id="line-419"> throw e;</span>
<span class="source-line-no">420</span><span id="line-420"> }</span>
<span class="source-line-no">421</span><span id="line-421"> }</span>
<span class="source-line-no">422</span><span id="line-422"> }</span>
<span class="source-line-no">423</span><span id="line-423"></span>
<span class="source-line-no">424</span><span id="line-424"> private FSDataOutputStream createFileWithRetries(final FileSystem fs,</span>
<span class="source-line-no">425</span><span id="line-425"> final Path hbckLockFilePath, final FsPermission defaultPerms) throws IOException {</span>
<span class="source-line-no">426</span><span id="line-426"> IOException exception = null;</span>
<span class="source-line-no">427</span><span id="line-427"> do {</span>
<span class="source-line-no">428</span><span id="line-428"> try {</span>
<span class="source-line-no">429</span><span id="line-429"> return CommonFSUtils.create(fs, hbckLockFilePath, defaultPerms, false);</span>
<span class="source-line-no">430</span><span id="line-430"> } catch (IOException ioe) {</span>
<span class="source-line-no">431</span><span id="line-431"> LOG.info("Failed to create lock file " + hbckLockFilePath.getName() + ", try="</span>
<span class="source-line-no">432</span><span id="line-432"> + (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());</span>
<span class="source-line-no">433</span><span id="line-433"> LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), ioe);</span>
<span class="source-line-no">434</span><span id="line-434"> try {</span>
<span class="source-line-no">435</span><span id="line-435"> exception = ioe;</span>
<span class="source-line-no">436</span><span id="line-436"> retryCounter.sleepUntilNextRetry();</span>
<span class="source-line-no">437</span><span id="line-437"> } catch (InterruptedException ie) {</span>
<span class="source-line-no">438</span><span id="line-438"> throw (InterruptedIOException) new InterruptedIOException(</span>
<span class="source-line-no">439</span><span id="line-439"> "Can't create lock file " + hbckLockFilePath.getName()).initCause(ie);</span>
<span class="source-line-no">440</span><span id="line-440"> }</span>
<span class="source-line-no">441</span><span id="line-441"> }</span>
<span class="source-line-no">442</span><span id="line-442"> } while (retryCounter.shouldRetry());</span>
<span class="source-line-no">443</span><span id="line-443"></span>
<span class="source-line-no">444</span><span id="line-444"> throw exception;</span>
<span class="source-line-no">445</span><span id="line-445"> }</span>
<span class="source-line-no">446</span><span id="line-446"> }</span>
<span class="source-line-no">447</span><span id="line-447"></span>
<span class="source-line-no">448</span><span id="line-448"> /**</span>
<span class="source-line-no">449</span><span id="line-449"> * This method maintains a lock using a file. If the creation fails we return null</span>
<span class="source-line-no">450</span><span id="line-450"> * @return FSDataOutputStream object corresponding to the newly opened lock file</span>
<span class="source-line-no">451</span><span id="line-451"> * @throws IOException if IO failure occurs</span>
<span class="source-line-no">452</span><span id="line-452"> */</span>
<span class="source-line-no">453</span><span id="line-453"> public static Pair&lt;Path, FSDataOutputStream&gt; checkAndMarkRunningHbck(Configuration conf,</span>
<span class="source-line-no">454</span><span id="line-454"> RetryCounter retryCounter) throws IOException {</span>
<span class="source-line-no">455</span><span id="line-455"> FileLockCallable callable = new FileLockCallable(conf, retryCounter);</span>
<span class="source-line-no">456</span><span id="line-456"> ExecutorService executor = Executors.newFixedThreadPool(1);</span>
<span class="source-line-no">457</span><span id="line-457"> FutureTask&lt;FSDataOutputStream&gt; futureTask = new FutureTask&lt;&gt;(callable);</span>
<span class="source-line-no">458</span><span id="line-458"> executor.execute(futureTask);</span>
<span class="source-line-no">459</span><span id="line-459"> final int timeoutInSeconds =</span>
<span class="source-line-no">460</span><span id="line-460"> conf.getInt("hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT);</span>
<span class="source-line-no">461</span><span id="line-461"> FSDataOutputStream stream = null;</span>
<span class="source-line-no">462</span><span id="line-462"> try {</span>
<span class="source-line-no">463</span><span id="line-463"> stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS);</span>
<span class="source-line-no">464</span><span id="line-464"> } catch (ExecutionException ee) {</span>
<span class="source-line-no">465</span><span id="line-465"> LOG.warn("Encountered exception when opening lock file", ee);</span>
<span class="source-line-no">466</span><span id="line-466"> } catch (InterruptedException ie) {</span>
<span class="source-line-no">467</span><span id="line-467"> LOG.warn("Interrupted when opening lock file", ie);</span>
<span class="source-line-no">468</span><span id="line-468"> Thread.currentThread().interrupt();</span>
<span class="source-line-no">469</span><span id="line-469"> } catch (TimeoutException exception) {</span>
<span class="source-line-no">470</span><span id="line-470"> // took too long to obtain lock</span>
<span class="source-line-no">471</span><span id="line-471"> LOG.warn("Took more than " + timeoutInSeconds + " seconds in obtaining lock");</span>
<span class="source-line-no">472</span><span id="line-472"> futureTask.cancel(true);</span>
<span class="source-line-no">473</span><span id="line-473"> } finally {</span>
<span class="source-line-no">474</span><span id="line-474"> executor.shutdownNow();</span>
<span class="source-line-no">475</span><span id="line-475"> }</span>
<span class="source-line-no">476</span><span id="line-476"> return new Pair&lt;Path, FSDataOutputStream&gt;(callable.getHbckLockPath(), stream);</span>
<span class="source-line-no">477</span><span id="line-477"> }</span>
<span class="source-line-no">478</span><span id="line-478"></span>
<span class="source-line-no">479</span><span id="line-479"> private void unlockHbck() {</span>
<span class="source-line-no">480</span><span id="line-480"> if (isExclusive() &amp;&amp; hbckLockCleanup.compareAndSet(true, false)) {</span>
<span class="source-line-no">481</span><span id="line-481"> RetryCounter retryCounter = lockFileRetryCounterFactory.create();</span>
<span class="source-line-no">482</span><span id="line-482"> do {</span>
<span class="source-line-no">483</span><span id="line-483"> try {</span>
<span class="source-line-no">484</span><span id="line-484"> Closeables.close(hbckOutFd, true);</span>
<span class="source-line-no">485</span><span id="line-485"> CommonFSUtils.delete(CommonFSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);</span>
<span class="source-line-no">486</span><span id="line-486"> LOG.info("Finishing hbck");</span>
<span class="source-line-no">487</span><span id="line-487"> return;</span>
<span class="source-line-no">488</span><span id="line-488"> } catch (IOException ioe) {</span>
<span class="source-line-no">489</span><span id="line-489"> LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try="</span>
<span class="source-line-no">490</span><span id="line-490"> + (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());</span>
<span class="source-line-no">491</span><span id="line-491"> LOG.debug("Failed to delete " + HBCK_LOCK_PATH, ioe);</span>
<span class="source-line-no">492</span><span id="line-492"> try {</span>
<span class="source-line-no">493</span><span id="line-493"> retryCounter.sleepUntilNextRetry();</span>
<span class="source-line-no">494</span><span id="line-494"> } catch (InterruptedException ie) {</span>
<span class="source-line-no">495</span><span id="line-495"> Thread.currentThread().interrupt();</span>
<span class="source-line-no">496</span><span id="line-496"> LOG.warn("Interrupted while deleting lock file" + HBCK_LOCK_PATH);</span>
<span class="source-line-no">497</span><span id="line-497"> return;</span>
<span class="source-line-no">498</span><span id="line-498"> }</span>
<span class="source-line-no">499</span><span id="line-499"> }</span>
<span class="source-line-no">500</span><span id="line-500"> } while (retryCounter.shouldRetry());</span>
<span class="source-line-no">501</span><span id="line-501"> }</span>
<span class="source-line-no">502</span><span id="line-502"> }</span>
<span class="source-line-no">503</span><span id="line-503"></span>
<span class="source-line-no">504</span><span id="line-504"> /**</span>
<span class="source-line-no">505</span><span id="line-505"> * To repair region consistency, one must call connect() in order to repair online state.</span>
<span class="source-line-no">506</span><span id="line-506"> */</span>
<span class="source-line-no">507</span><span id="line-507"> public void connect() throws IOException {</span>
<span class="source-line-no">508</span><span id="line-508"></span>
<span class="source-line-no">509</span><span id="line-509"> if (isExclusive()) {</span>
<span class="source-line-no">510</span><span id="line-510"> // Grab the lock</span>
<span class="source-line-no">511</span><span id="line-511"> Pair&lt;Path, FSDataOutputStream&gt; pair =</span>
<span class="source-line-no">512</span><span id="line-512"> checkAndMarkRunningHbck(getConf(), this.lockFileRetryCounterFactory.create());</span>
<span class="source-line-no">513</span><span id="line-513"> HBCK_LOCK_PATH = pair.getFirst();</span>
<span class="source-line-no">514</span><span id="line-514"> this.hbckOutFd = pair.getSecond();</span>
<span class="source-line-no">515</span><span id="line-515"> if (hbckOutFd == null) {</span>
<span class="source-line-no">516</span><span id="line-516"> setRetCode(-1);</span>
<span class="source-line-no">517</span><span id="line-517"> LOG.error("Another instance of hbck is fixing HBase, exiting this instance. "</span>
<span class="source-line-no">518</span><span id="line-518"> + "[If you are sure no other instance is running, delete the lock file " + HBCK_LOCK_PATH</span>
<span class="source-line-no">519</span><span id="line-519"> + " and rerun the tool]");</span>
<span class="source-line-no">520</span><span id="line-520"> throw new IOException("Duplicate hbck - Abort");</span>
<span class="source-line-no">521</span><span id="line-521"> }</span>
<span class="source-line-no">522</span><span id="line-522"></span>
<span class="source-line-no">523</span><span id="line-523"> // Make sure to cleanup the lock</span>
<span class="source-line-no">524</span><span id="line-524"> hbckLockCleanup.set(true);</span>
<span class="source-line-no">525</span><span id="line-525"> }</span>
<span class="source-line-no">526</span><span id="line-526"></span>
<span class="source-line-no">527</span><span id="line-527"> // Add a shutdown hook to this thread, in case user tries to</span>
<span class="source-line-no">528</span><span id="line-528"> // kill the hbck with a ctrl-c, we want to cleanup the lock so that</span>
<span class="source-line-no">529</span><span id="line-529"> // it is available for further calls</span>
<span class="source-line-no">530</span><span id="line-530"> Runtime.getRuntime().addShutdownHook(new Thread() {</span>
<span class="source-line-no">531</span><span id="line-531"> @Override</span>
<span class="source-line-no">532</span><span id="line-532"> public void run() {</span>
<span class="source-line-no">533</span><span id="line-533"> IOUtils.closeQuietly(HBaseFsck.this, e -&gt; LOG.warn("", e));</span>
<span class="source-line-no">534</span><span id="line-534"> cleanupHbckZnode();</span>
<span class="source-line-no">535</span><span id="line-535"> unlockHbck();</span>
<span class="source-line-no">536</span><span id="line-536"> }</span>
<span class="source-line-no">537</span><span id="line-537"> });</span>
<span class="source-line-no">538</span><span id="line-538"></span>
<span class="source-line-no">539</span><span id="line-539"> LOG.info("Launching hbck");</span>
<span class="source-line-no">540</span><span id="line-540"></span>
<span class="source-line-no">541</span><span id="line-541"> connection = ConnectionFactory.createConnection(getConf());</span>
<span class="source-line-no">542</span><span id="line-542"> admin = connection.getAdmin();</span>
<span class="source-line-no">543</span><span id="line-543"> meta = connection.getTable(TableName.META_TABLE_NAME);</span>
<span class="source-line-no">544</span><span id="line-544"> status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS, Option.DEAD_SERVERS,</span>
<span class="source-line-no">545</span><span id="line-545"> Option.MASTER, Option.BACKUP_MASTERS, Option.REGIONS_IN_TRANSITION, Option.HBASE_VERSION));</span>
<span class="source-line-no">546</span><span id="line-546"> }</span>
<span class="source-line-no">547</span><span id="line-547"></span>
<span class="source-line-no">548</span><span id="line-548"> /**</span>
<span class="source-line-no">549</span><span id="line-549"> * Get deployed regions according to the region servers.</span>
<span class="source-line-no">550</span><span id="line-550"> */</span>
<span class="source-line-no">551</span><span id="line-551"> private void loadDeployedRegions() throws IOException, InterruptedException {</span>
<span class="source-line-no">552</span><span id="line-552"> // From the master, get a list of all known live region servers</span>
<span class="source-line-no">553</span><span id="line-553"> Collection&lt;ServerName&gt; regionServers = status.getLiveServerMetrics().keySet();</span>
<span class="source-line-no">554</span><span id="line-554"> errors.print("Number of live region servers: " + regionServers.size());</span>
<span class="source-line-no">555</span><span id="line-555"> if (details) {</span>
<span class="source-line-no">556</span><span id="line-556"> for (ServerName rsinfo : regionServers) {</span>
<span class="source-line-no">557</span><span id="line-557"> errors.print(" " + rsinfo.getServerName());</span>
<span class="source-line-no">558</span><span id="line-558"> }</span>
<span class="source-line-no">559</span><span id="line-559"> }</span>
<span class="source-line-no">560</span><span id="line-560"></span>
<span class="source-line-no">561</span><span id="line-561"> // From the master, get a list of all dead region servers</span>
<span class="source-line-no">562</span><span id="line-562"> Collection&lt;ServerName&gt; deadRegionServers = status.getDeadServerNames();</span>
<span class="source-line-no">563</span><span id="line-563"> errors.print("Number of dead region servers: " + deadRegionServers.size());</span>
<span class="source-line-no">564</span><span id="line-564"> if (details) {</span>
<span class="source-line-no">565</span><span id="line-565"> for (ServerName name : deadRegionServers) {</span>
<span class="source-line-no">566</span><span id="line-566"> errors.print(" " + name);</span>
<span class="source-line-no">567</span><span id="line-567"> }</span>
<span class="source-line-no">568</span><span id="line-568"> }</span>
<span class="source-line-no">569</span><span id="line-569"></span>
<span class="source-line-no">570</span><span id="line-570"> // Print the current master name and state</span>
<span class="source-line-no">571</span><span id="line-571"> errors.print("Master: " + status.getMasterName());</span>
<span class="source-line-no">572</span><span id="line-572"></span>
<span class="source-line-no">573</span><span id="line-573"> // Print the list of all backup masters</span>
<span class="source-line-no">574</span><span id="line-574"> Collection&lt;ServerName&gt; backupMasters = status.getBackupMasterNames();</span>
<span class="source-line-no">575</span><span id="line-575"> errors.print("Number of backup masters: " + backupMasters.size());</span>
<span class="source-line-no">576</span><span id="line-576"> if (details) {</span>
<span class="source-line-no">577</span><span id="line-577"> for (ServerName name : backupMasters) {</span>
<span class="source-line-no">578</span><span id="line-578"> errors.print(" " + name);</span>
<span class="source-line-no">579</span><span id="line-579"> }</span>
<span class="source-line-no">580</span><span id="line-580"> }</span>
<span class="source-line-no">581</span><span id="line-581"></span>
<span class="source-line-no">582</span><span id="line-582"> errors.print("Average load: " + status.getAverageLoad());</span>
<span class="source-line-no">583</span><span id="line-583"> errors.print("Number of requests: " + status.getRequestCount());</span>
<span class="source-line-no">584</span><span id="line-584"> errors.print("Number of regions: " + status.getRegionCount());</span>
<span class="source-line-no">585</span><span id="line-585"></span>
<span class="source-line-no">586</span><span id="line-586"> List&lt;RegionState&gt; rits = status.getRegionStatesInTransition();</span>
<span class="source-line-no">587</span><span id="line-587"> errors.print("Number of regions in transition: " + rits.size());</span>
<span class="source-line-no">588</span><span id="line-588"> if (details) {</span>
<span class="source-line-no">589</span><span id="line-589"> for (RegionState state : rits) {</span>
<span class="source-line-no">590</span><span id="line-590"> errors.print(" " + state.toDescriptiveString());</span>
<span class="source-line-no">591</span><span id="line-591"> }</span>
<span class="source-line-no">592</span><span id="line-592"> }</span>
<span class="source-line-no">593</span><span id="line-593"></span>
<span class="source-line-no">594</span><span id="line-594"> // Determine what's deployed</span>
<span class="source-line-no">595</span><span id="line-595"> processRegionServers(regionServers);</span>
<span class="source-line-no">596</span><span id="line-596"> }</span>
<span class="source-line-no">597</span><span id="line-597"></span>
<span class="source-line-no">598</span><span id="line-598"> /**</span>
<span class="source-line-no">599</span><span id="line-599"> * Clear the current state of hbck.</span>
<span class="source-line-no">600</span><span id="line-600"> */</span>
<span class="source-line-no">601</span><span id="line-601"> private void clearState() {</span>
<span class="source-line-no">602</span><span id="line-602"> // Make sure regionInfo is empty before starting</span>
<span class="source-line-no">603</span><span id="line-603"> fixes = 0;</span>
<span class="source-line-no">604</span><span id="line-604"> regionInfoMap.clear();</span>
<span class="source-line-no">605</span><span id="line-605"> emptyRegionInfoQualifiers.clear();</span>
<span class="source-line-no">606</span><span id="line-606"> tableStates.clear();</span>
<span class="source-line-no">607</span><span id="line-607"> errors.clear();</span>
<span class="source-line-no">608</span><span id="line-608"> tablesInfo.clear();</span>
<span class="source-line-no">609</span><span id="line-609"> orphanHdfsDirs.clear();</span>
<span class="source-line-no">610</span><span id="line-610"> skippedRegions.clear();</span>
<span class="source-line-no">611</span><span id="line-611"> }</span>
<span class="source-line-no">612</span><span id="line-612"></span>
<span class="source-line-no">613</span><span id="line-613"> /**</span>
<span class="source-line-no">614</span><span id="line-614"> * This repair method analyzes hbase data in hdfs and repairs it to satisfy the table integrity</span>
<span class="source-line-no">615</span><span id="line-615"> * rules. HBase doesn't need to be online for this operation to work.</span>
<span class="source-line-no">616</span><span id="line-616"> */</span>
<span class="source-line-no">617</span><span id="line-617"> public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {</span>
<span class="source-line-no">618</span><span id="line-618"> // Initial pass to fix orphans.</span>
<span class="source-line-no">619</span><span id="line-619"> if (</span>
<span class="source-line-no">620</span><span id="line-620"> shouldCheckHdfs() &amp;&amp; (shouldFixHdfsOrphans() || shouldFixHdfsHoles()</span>
<span class="source-line-no">621</span><span id="line-621"> || shouldFixHdfsOverlaps() || shouldFixTableOrphans())</span>
<span class="source-line-no">622</span><span id="line-622"> ) {</span>
<span class="source-line-no">623</span><span id="line-623"> LOG.info("Loading regioninfos HDFS");</span>
<span class="source-line-no">624</span><span id="line-624"> // if nothing is happening this should always complete in two iterations.</span>
<span class="source-line-no">625</span><span id="line-625"> int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);</span>
<span class="source-line-no">626</span><span id="line-626"> int curIter = 0;</span>
<span class="source-line-no">627</span><span id="line-627"> do {</span>
<span class="source-line-no">628</span><span id="line-628"> clearState(); // clears hbck state and reset fixes to 0 and.</span>
<span class="source-line-no">629</span><span id="line-629"> // repair what's on HDFS</span>
<span class="source-line-no">630</span><span id="line-630"> restoreHdfsIntegrity();</span>
<span class="source-line-no">631</span><span id="line-631"> curIter++;// limit the number of iterations.</span>
<span class="source-line-no">632</span><span id="line-632"> } while (fixes &gt; 0 &amp;&amp; curIter &lt;= maxIterations);</span>
<span class="source-line-no">633</span><span id="line-633"></span>
<span class="source-line-no">634</span><span id="line-634"> // Repairs should be done in the first iteration and verification in the second.</span>
<span class="source-line-no">635</span><span id="line-635"> // If there are more than 2 passes, something funny has happened.</span>
<span class="source-line-no">636</span><span id="line-636"> if (curIter &gt; 2) {</span>
<span class="source-line-no">637</span><span id="line-637"> if (curIter == maxIterations) {</span>
<span class="source-line-no">638</span><span id="line-638"> LOG.warn("Exiting integrity repairs after max " + curIter + " iterations. "</span>
<span class="source-line-no">639</span><span id="line-639"> + "Tables integrity may not be fully repaired!");</span>
<span class="source-line-no">640</span><span id="line-640"> } else {</span>
<span class="source-line-no">641</span><span id="line-641"> LOG.info("Successfully exiting integrity repairs after " + curIter + " iterations");</span>
<span class="source-line-no">642</span><span id="line-642"> }</span>
<span class="source-line-no">643</span><span id="line-643"> }</span>
<span class="source-line-no">644</span><span id="line-644"> }</span>
<span class="source-line-no">645</span><span id="line-645"> }</span>
<span class="source-line-no">646</span><span id="line-646"></span>
<span class="source-line-no">647</span><span id="line-647"> /**</span>
<span class="source-line-no">648</span><span id="line-648"> * This repair method requires the cluster to be online since it contacts region servers and the</span>
<span class="source-line-no">649</span><span id="line-649"> * masters. It makes each region's state in HDFS, in hbase:meta, and deployments consistent.</span>
<span class="source-line-no">650</span><span id="line-650"> * @return If &amp;gt; 0 , number of errors detected, if &amp;lt; 0 there was an unrecoverable error. If</span>
<span class="source-line-no">651</span><span id="line-651"> * 0, we have a clean hbase.</span>
<span class="source-line-no">652</span><span id="line-652"> */</span>
<span class="source-line-no">653</span><span id="line-653"> public int onlineConsistencyRepair() throws IOException, KeeperException, InterruptedException {</span>
<span class="source-line-no">654</span><span id="line-654"></span>
<span class="source-line-no">655</span><span id="line-655"> // get regions according to what is online on each RegionServer</span>
<span class="source-line-no">656</span><span id="line-656"> loadDeployedRegions();</span>
<span class="source-line-no">657</span><span id="line-657"> // check whether hbase:meta is deployed and online</span>
<span class="source-line-no">658</span><span id="line-658"> recordMetaRegion();</span>
<span class="source-line-no">659</span><span id="line-659"> // Report inconsistencies if there are any unknown servers.</span>
<span class="source-line-no">660</span><span id="line-660"> reportUnknownServers();</span>
<span class="source-line-no">661</span><span id="line-661"> // Check if hbase:meta is found only once and in the right place</span>
<span class="source-line-no">662</span><span id="line-662"> if (!checkMetaRegion()) {</span>
<span class="source-line-no">663</span><span id="line-663"> String errorMsg = "hbase:meta table is not consistent. ";</span>
<span class="source-line-no">664</span><span id="line-664"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">665</span><span id="line-665"> errorMsg += "HBCK will try fixing it. Rerun once hbase:meta is back to consistent state.";</span>
<span class="source-line-no">666</span><span id="line-666"> } else {</span>
<span class="source-line-no">667</span><span id="line-667"> errorMsg += "Run HBCK with proper fix options to fix hbase:meta inconsistency.";</span>
<span class="source-line-no">668</span><span id="line-668"> }</span>
<span class="source-line-no">669</span><span id="line-669"> errors.reportError(errorMsg + " Exiting...");</span>
<span class="source-line-no">670</span><span id="line-670"> return -2;</span>
<span class="source-line-no">671</span><span id="line-671"> }</span>
<span class="source-line-no">672</span><span id="line-672"> // Not going with further consistency check for tables when hbase:meta itself is not consistent.</span>
<span class="source-line-no">673</span><span id="line-673"> LOG.info("Loading regionsinfo from the hbase:meta table");</span>
<span class="source-line-no">674</span><span id="line-674"> boolean success = loadMetaEntries();</span>
<span class="source-line-no">675</span><span id="line-675"> if (!success) return -1;</span>
<span class="source-line-no">676</span><span id="line-676"></span>
<span class="source-line-no">677</span><span id="line-677"> // Empty cells in hbase:meta?</span>
<span class="source-line-no">678</span><span id="line-678"> reportEmptyMetaCells();</span>
<span class="source-line-no">679</span><span id="line-679"></span>
<span class="source-line-no">680</span><span id="line-680"> // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from hbase:meta</span>
<span class="source-line-no">681</span><span id="line-681"> if (shouldFixEmptyMetaCells()) {</span>
<span class="source-line-no">682</span><span id="line-682"> fixEmptyMetaCells();</span>
<span class="source-line-no">683</span><span id="line-683"> }</span>
<span class="source-line-no">684</span><span id="line-684"></span>
<span class="source-line-no">685</span><span id="line-685"> // get a list of all tables that have not changed recently.</span>
<span class="source-line-no">686</span><span id="line-686"> if (!checkMetaOnly) {</span>
<span class="source-line-no">687</span><span id="line-687"> reportTablesInFlux();</span>
<span class="source-line-no">688</span><span id="line-688"> }</span>
<span class="source-line-no">689</span><span id="line-689"></span>
<span class="source-line-no">690</span><span id="line-690"> // Get disabled tables states</span>
<span class="source-line-no">691</span><span id="line-691"> loadTableStates();</span>
<span class="source-line-no">692</span><span id="line-692"></span>
<span class="source-line-no">693</span><span id="line-693"> // load regiondirs and regioninfos from HDFS</span>
<span class="source-line-no">694</span><span id="line-694"> if (shouldCheckHdfs()) {</span>
<span class="source-line-no">695</span><span id="line-695"> LOG.info("Loading region directories from HDFS");</span>
<span class="source-line-no">696</span><span id="line-696"> loadHdfsRegionDirs();</span>
<span class="source-line-no">697</span><span id="line-697"> LOG.info("Loading region information from HDFS");</span>
<span class="source-line-no">698</span><span id="line-698"> loadHdfsRegionInfos();</span>
<span class="source-line-no">699</span><span id="line-699"> }</span>
<span class="source-line-no">700</span><span id="line-700"></span>
<span class="source-line-no">701</span><span id="line-701"> // fix the orphan tables</span>
<span class="source-line-no">702</span><span id="line-702"> fixOrphanTables();</span>
<span class="source-line-no">703</span><span id="line-703"></span>
<span class="source-line-no">704</span><span id="line-704"> LOG.info("Checking and fixing region consistency");</span>
<span class="source-line-no">705</span><span id="line-705"> // Check and fix consistency</span>
<span class="source-line-no">706</span><span id="line-706"> checkAndFixConsistency();</span>
<span class="source-line-no">707</span><span id="line-707"></span>
<span class="source-line-no">708</span><span id="line-708"> // Check integrity (does not fix)</span>
<span class="source-line-no">709</span><span id="line-709"> checkIntegrity();</span>
<span class="source-line-no">710</span><span id="line-710"> return errors.getErrorList().size();</span>
<span class="source-line-no">711</span><span id="line-711"> }</span>
<span class="source-line-no">712</span><span id="line-712"></span>
<span class="source-line-no">713</span><span id="line-713"> private void reportUnknownServers() throws IOException {</span>
<span class="source-line-no">714</span><span id="line-714"> List&lt;ServerName&gt; unknownServers = admin.listUnknownServers();</span>
<span class="source-line-no">715</span><span id="line-715"> if (!unknownServers.isEmpty()) {</span>
<span class="source-line-no">716</span><span id="line-716"> unknownServers.stream().forEach(serverName -&gt; {</span>
<span class="source-line-no">717</span><span id="line-717"> errors.reportError(ERROR_CODE.UNKNOWN_SERVER,</span>
<span class="source-line-no">718</span><span id="line-718"> "Found unknown server, some of the regions held by this server may not get assigned. "</span>
<span class="source-line-no">719</span><span id="line-719"> + String.format("Use HBCK2 scheduleRecoveries %s to recover.", serverName));</span>
<span class="source-line-no">720</span><span id="line-720"> });</span>
<span class="source-line-no">721</span><span id="line-721"> }</span>
<span class="source-line-no">722</span><span id="line-722"> }</span>
<span class="source-line-no">723</span><span id="line-723"></span>
<span class="source-line-no">724</span><span id="line-724"> /**</span>
<span class="source-line-no">725</span><span id="line-725"> * This method maintains an ephemeral znode. If the creation fails we return false or throw</span>
<span class="source-line-no">726</span><span id="line-726"> * exception</span>
<span class="source-line-no">727</span><span id="line-727"> * @return true if creating znode succeeds; false otherwise</span>
<span class="source-line-no">728</span><span id="line-728"> * @throws IOException if IO failure occurs</span>
<span class="source-line-no">729</span><span id="line-729"> */</span>
<span class="source-line-no">730</span><span id="line-730"> private boolean setMasterInMaintenanceMode() throws IOException {</span>
<span class="source-line-no">731</span><span id="line-731"> RetryCounter retryCounter = createZNodeRetryCounterFactory.create();</span>
<span class="source-line-no">732</span><span id="line-732"> hbckEphemeralNodePath = ZNodePaths.joinZNode(zkw.getZNodePaths().masterMaintZNode,</span>
<span class="source-line-no">733</span><span id="line-733"> "hbck-" + Long.toString(EnvironmentEdgeManager.currentTime()));</span>
<span class="source-line-no">734</span><span id="line-734"> do {</span>
<span class="source-line-no">735</span><span id="line-735"> try {</span>
<span class="source-line-no">736</span><span id="line-736"> hbckZodeCreated = ZKUtil.createEphemeralNodeAndWatch(zkw, hbckEphemeralNodePath, null);</span>
<span class="source-line-no">737</span><span id="line-737"> if (hbckZodeCreated) {</span>
<span class="source-line-no">738</span><span id="line-738"> break;</span>
<span class="source-line-no">739</span><span id="line-739"> }</span>
<span class="source-line-no">740</span><span id="line-740"> } catch (KeeperException e) {</span>
<span class="source-line-no">741</span><span id="line-741"> if (retryCounter.getAttemptTimes() &gt;= retryCounter.getMaxAttempts()) {</span>
<span class="source-line-no">742</span><span id="line-742"> throw new IOException("Can't create znode " + hbckEphemeralNodePath, e);</span>
<span class="source-line-no">743</span><span id="line-743"> }</span>
<span class="source-line-no">744</span><span id="line-744"> // fall through and retry</span>
<span class="source-line-no">745</span><span id="line-745"> }</span>
<span class="source-line-no">746</span><span id="line-746"></span>
<span class="source-line-no">747</span><span id="line-747"> LOG.warn("Fail to create znode " + hbckEphemeralNodePath + ", try="</span>
<span class="source-line-no">748</span><span id="line-748"> + (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());</span>
<span class="source-line-no">749</span><span id="line-749"></span>
<span class="source-line-no">750</span><span id="line-750"> try {</span>
<span class="source-line-no">751</span><span id="line-751"> retryCounter.sleepUntilNextRetry();</span>
<span class="source-line-no">752</span><span id="line-752"> } catch (InterruptedException ie) {</span>
<span class="source-line-no">753</span><span id="line-753"> throw (InterruptedIOException) new InterruptedIOException(</span>
<span class="source-line-no">754</span><span id="line-754"> "Can't create znode " + hbckEphemeralNodePath).initCause(ie);</span>
<span class="source-line-no">755</span><span id="line-755"> }</span>
<span class="source-line-no">756</span><span id="line-756"> } while (retryCounter.shouldRetry());</span>
<span class="source-line-no">757</span><span id="line-757"> return hbckZodeCreated;</span>
<span class="source-line-no">758</span><span id="line-758"> }</span>
<span class="source-line-no">759</span><span id="line-759"></span>
<span class="source-line-no">760</span><span id="line-760"> private void cleanupHbckZnode() {</span>
<span class="source-line-no">761</span><span id="line-761"> try {</span>
<span class="source-line-no">762</span><span id="line-762"> if (zkw != null &amp;&amp; hbckZodeCreated) {</span>
<span class="source-line-no">763</span><span id="line-763"> ZKUtil.deleteNode(zkw, hbckEphemeralNodePath);</span>
<span class="source-line-no">764</span><span id="line-764"> hbckZodeCreated = false;</span>
<span class="source-line-no">765</span><span id="line-765"> }</span>
<span class="source-line-no">766</span><span id="line-766"> } catch (KeeperException e) {</span>
<span class="source-line-no">767</span><span id="line-767"> // Ignore</span>
<span class="source-line-no">768</span><span id="line-768"> if (!e.code().equals(KeeperException.Code.NONODE)) {</span>
<span class="source-line-no">769</span><span id="line-769"> LOG.warn("Delete HBCK znode " + hbckEphemeralNodePath + " failed ", e);</span>
<span class="source-line-no">770</span><span id="line-770"> }</span>
<span class="source-line-no">771</span><span id="line-771"> }</span>
<span class="source-line-no">772</span><span id="line-772"> }</span>
<span class="source-line-no">773</span><span id="line-773"></span>
<span class="source-line-no">774</span><span id="line-774"> /**</span>
<span class="source-line-no">775</span><span id="line-775"> * Contacts the master and prints out cluster-wide information</span>
<span class="source-line-no">776</span><span id="line-776"> * @return 0 on success, non-zero on failure</span>
<span class="source-line-no">777</span><span id="line-777"> */</span>
<span class="source-line-no">778</span><span id="line-778"> public int onlineHbck()</span>
<span class="source-line-no">779</span><span id="line-779"> throws IOException, KeeperException, InterruptedException, ReplicationException {</span>
<span class="source-line-no">780</span><span id="line-780"> // print hbase server version</span>
<span class="source-line-no">781</span><span id="line-781"> errors.print("Version: " + status.getHBaseVersion());</span>
<span class="source-line-no">782</span><span id="line-782"></span>
<span class="source-line-no">783</span><span id="line-783"> // Clean start</span>
<span class="source-line-no">784</span><span id="line-784"> clearState();</span>
<span class="source-line-no">785</span><span id="line-785"> // Do offline check and repair first</span>
<span class="source-line-no">786</span><span id="line-786"> offlineHdfsIntegrityRepair();</span>
<span class="source-line-no">787</span><span id="line-787"> offlineReferenceFileRepair();</span>
<span class="source-line-no">788</span><span id="line-788"> offlineHLinkFileRepair();</span>
<span class="source-line-no">789</span><span id="line-789"> // If Master runs maintenance tasks (such as balancer, catalog janitor, etc) during online</span>
<span class="source-line-no">790</span><span id="line-790"> // hbck, it is likely that hbck would be misled and report transient errors. Therefore, it</span>
<span class="source-line-no">791</span><span id="line-791"> // is better to set Master into maintenance mode during online hbck.</span>
<span class="source-line-no">792</span><span id="line-792"> //</span>
<span class="source-line-no">793</span><span id="line-793"> if (!setMasterInMaintenanceMode()) {</span>
<span class="source-line-no">794</span><span id="line-794"> LOG.warn("HBCK is running while master is not in maintenance mode, you might see transient "</span>
<span class="source-line-no">795</span><span id="line-795"> + "error. Please run HBCK multiple times to reduce the chance of transient error.");</span>
<span class="source-line-no">796</span><span id="line-796"> }</span>
<span class="source-line-no">797</span><span id="line-797"></span>
<span class="source-line-no">798</span><span id="line-798"> onlineConsistencyRepair();</span>
<span class="source-line-no">799</span><span id="line-799"></span>
<span class="source-line-no">800</span><span id="line-800"> if (checkRegionBoundaries) {</span>
<span class="source-line-no">801</span><span id="line-801"> checkRegionBoundaries();</span>
<span class="source-line-no">802</span><span id="line-802"> }</span>
<span class="source-line-no">803</span><span id="line-803"></span>
<span class="source-line-no">804</span><span id="line-804"> checkAndFixReplication();</span>
<span class="source-line-no">805</span><span id="line-805"></span>
<span class="source-line-no">806</span><span id="line-806"> cleanReplicationBarrier();</span>
<span class="source-line-no">807</span><span id="line-807"></span>
<span class="source-line-no">808</span><span id="line-808"> // Remove the hbck znode</span>
<span class="source-line-no">809</span><span id="line-809"> cleanupHbckZnode();</span>
<span class="source-line-no">810</span><span id="line-810"></span>
<span class="source-line-no">811</span><span id="line-811"> // Remove the hbck lock</span>
<span class="source-line-no">812</span><span id="line-812"> unlockHbck();</span>
<span class="source-line-no">813</span><span id="line-813"></span>
<span class="source-line-no">814</span><span id="line-814"> // Print table summary</span>
<span class="source-line-no">815</span><span id="line-815"> printTableSummary(tablesInfo);</span>
<span class="source-line-no">816</span><span id="line-816"> return errors.summarize();</span>
<span class="source-line-no">817</span><span id="line-817"> }</span>
<span class="source-line-no">818</span><span id="line-818"></span>
<span class="source-line-no">819</span><span id="line-819"> public static byte[] keyOnly(byte[] b) {</span>
<span class="source-line-no">820</span><span id="line-820"> if (b == null) return b;</span>
<span class="source-line-no">821</span><span id="line-821"> int rowlength = Bytes.toShort(b, 0);</span>
<span class="source-line-no">822</span><span id="line-822"> byte[] result = new byte[rowlength];</span>
<span class="source-line-no">823</span><span id="line-823"> System.arraycopy(b, Bytes.SIZEOF_SHORT, result, 0, rowlength);</span>
<span class="source-line-no">824</span><span id="line-824"> return result;</span>
<span class="source-line-no">825</span><span id="line-825"> }</span>
<span class="source-line-no">826</span><span id="line-826"></span>
<span class="source-line-no">827</span><span id="line-827"> @Override</span>
<span class="source-line-no">828</span><span id="line-828"> public void close() throws IOException {</span>
<span class="source-line-no">829</span><span id="line-829"> try {</span>
<span class="source-line-no">830</span><span id="line-830"> cleanupHbckZnode();</span>
<span class="source-line-no">831</span><span id="line-831"> unlockHbck();</span>
<span class="source-line-no">832</span><span id="line-832"> } catch (Exception io) {</span>
<span class="source-line-no">833</span><span id="line-833"> LOG.warn(io.toString(), io);</span>
<span class="source-line-no">834</span><span id="line-834"> } finally {</span>
<span class="source-line-no">835</span><span id="line-835"> if (zkw != null) {</span>
<span class="source-line-no">836</span><span id="line-836"> zkw.close();</span>
<span class="source-line-no">837</span><span id="line-837"> zkw = null;</span>
<span class="source-line-no">838</span><span id="line-838"> }</span>
<span class="source-line-no">839</span><span id="line-839"> IOUtils.closeQuietly(admin, e -&gt; LOG.warn("", e));</span>
<span class="source-line-no">840</span><span id="line-840"> IOUtils.closeQuietly(meta, e -&gt; LOG.warn("", e));</span>
<span class="source-line-no">841</span><span id="line-841"> IOUtils.closeQuietly(connection, e -&gt; LOG.warn("", e));</span>
<span class="source-line-no">842</span><span id="line-842"> }</span>
<span class="source-line-no">843</span><span id="line-843"> }</span>
<span class="source-line-no">844</span><span id="line-844"></span>
<span class="source-line-no">845</span><span id="line-845"> private static class RegionBoundariesInformation {</span>
<span class="source-line-no">846</span><span id="line-846"> public byte[] regionName;</span>
<span class="source-line-no">847</span><span id="line-847"> public byte[] metaFirstKey;</span>
<span class="source-line-no">848</span><span id="line-848"> public byte[] metaLastKey;</span>
<span class="source-line-no">849</span><span id="line-849"> public byte[] storesFirstKey;</span>
<span class="source-line-no">850</span><span id="line-850"> public byte[] storesLastKey;</span>
<span class="source-line-no">851</span><span id="line-851"></span>
<span class="source-line-no">852</span><span id="line-852"> @Override</span>
<span class="source-line-no">853</span><span id="line-853"> public String toString() {</span>
<span class="source-line-no">854</span><span id="line-854"> return "regionName=" + Bytes.toStringBinary(regionName) + "\nmetaFirstKey="</span>
<span class="source-line-no">855</span><span id="line-855"> + Bytes.toStringBinary(metaFirstKey) + "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey)</span>
<span class="source-line-no">856</span><span id="line-856"> + "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) + "\nstoresLastKey="</span>
<span class="source-line-no">857</span><span id="line-857"> + Bytes.toStringBinary(storesLastKey);</span>
<span class="source-line-no">858</span><span id="line-858"> }</span>
<span class="source-line-no">859</span><span id="line-859"> }</span>
<span class="source-line-no">860</span><span id="line-860"></span>
<span class="source-line-no">861</span><span id="line-861"> public void checkRegionBoundaries() {</span>
<span class="source-line-no">862</span><span id="line-862"> try {</span>
<span class="source-line-no">863</span><span id="line-863"> ByteArrayComparator comparator = new ByteArrayComparator();</span>
<span class="source-line-no">864</span><span id="line-864"> List&lt;RegionInfo&gt; regions = MetaTableAccessor.getAllRegions(connection, true);</span>
<span class="source-line-no">865</span><span id="line-865"> final RegionBoundariesInformation currentRegionBoundariesInformation =</span>
<span class="source-line-no">866</span><span id="line-866"> new RegionBoundariesInformation();</span>
<span class="source-line-no">867</span><span id="line-867"> Path hbaseRoot = CommonFSUtils.getRootDir(getConf());</span>
<span class="source-line-no">868</span><span id="line-868"> for (RegionInfo regionInfo : regions) {</span>
<span class="source-line-no">869</span><span id="line-869"> Path tableDir = CommonFSUtils.getTableDir(hbaseRoot, regionInfo.getTable());</span>
<span class="source-line-no">870</span><span id="line-870"> currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();</span>
<span class="source-line-no">871</span><span id="line-871"> // For each region, get the start and stop key from the META and compare them to the</span>
<span class="source-line-no">872</span><span id="line-872"> // same information from the Stores.</span>
<span class="source-line-no">873</span><span id="line-873"> Path path = new Path(tableDir, regionInfo.getEncodedName());</span>
<span class="source-line-no">874</span><span id="line-874"> FileSystem fs = path.getFileSystem(getConf());</span>
<span class="source-line-no">875</span><span id="line-875"> FileStatus[] files = fs.listStatus(path);</span>
<span class="source-line-no">876</span><span id="line-876"> // For all the column families in this region...</span>
<span class="source-line-no">877</span><span id="line-877"> byte[] storeFirstKey = null;</span>
<span class="source-line-no">878</span><span id="line-878"> byte[] storeLastKey = null;</span>
<span class="source-line-no">879</span><span id="line-879"> for (FileStatus file : files) {</span>
<span class="source-line-no">880</span><span id="line-880"> String fileName = file.getPath().toString();</span>
<span class="source-line-no">881</span><span id="line-881"> fileName = fileName.substring(fileName.lastIndexOf("/") + 1);</span>
<span class="source-line-no">882</span><span id="line-882"> if (!fileName.startsWith(".") &amp;&amp; !fileName.endsWith("recovered.edits")) {</span>
<span class="source-line-no">883</span><span id="line-883"> FileStatus[] storeFiles = fs.listStatus(file.getPath());</span>
<span class="source-line-no">884</span><span id="line-884"> // For all the stores in this column family.</span>
<span class="source-line-no">885</span><span id="line-885"> for (FileStatus storeFile : storeFiles) {</span>
<span class="source-line-no">886</span><span id="line-886"> HFile.Reader reader =</span>
<span class="source-line-no">887</span><span id="line-887"> HFile.createReader(fs, storeFile.getPath(), CacheConfig.DISABLED, true, getConf());</span>
<span class="source-line-no">888</span><span id="line-888"> if (</span>
<span class="source-line-no">889</span><span id="line-889"> (reader.getFirstKey() != null)</span>
<span class="source-line-no">890</span><span id="line-890"> &amp;&amp; ((storeFirstKey == null) || (comparator.compare(storeFirstKey,</span>
<span class="source-line-no">891</span><span id="line-891"> ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey()) &gt; 0))</span>
<span class="source-line-no">892</span><span id="line-892"> ) {</span>
<span class="source-line-no">893</span><span id="line-893"> storeFirstKey = ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey();</span>
<span class="source-line-no">894</span><span id="line-894"> }</span>
<span class="source-line-no">895</span><span id="line-895"> if (</span>
<span class="source-line-no">896</span><span id="line-896"> (reader.getLastKey() != null)</span>
<span class="source-line-no">897</span><span id="line-897"> &amp;&amp; ((storeLastKey == null) || (comparator.compare(storeLastKey,</span>
<span class="source-line-no">898</span><span id="line-898"> ((KeyValue.KeyOnlyKeyValue) reader.getLastKey().get()).getKey())) &lt; 0)</span>
<span class="source-line-no">899</span><span id="line-899"> ) {</span>
<span class="source-line-no">900</span><span id="line-900"> storeLastKey = ((KeyValue.KeyOnlyKeyValue) reader.getLastKey().get()).getKey();</span>
<span class="source-line-no">901</span><span id="line-901"> }</span>
<span class="source-line-no">902</span><span id="line-902"> reader.close();</span>
<span class="source-line-no">903</span><span id="line-903"> }</span>
<span class="source-line-no">904</span><span id="line-904"> }</span>
<span class="source-line-no">905</span><span id="line-905"> }</span>
<span class="source-line-no">906</span><span id="line-906"> currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();</span>
<span class="source-line-no">907</span><span id="line-907"> currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();</span>
<span class="source-line-no">908</span><span id="line-908"> currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);</span>
<span class="source-line-no">909</span><span id="line-909"> currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);</span>
<span class="source-line-no">910</span><span id="line-910"> if (currentRegionBoundariesInformation.metaFirstKey.length == 0)</span>
<span class="source-line-no">911</span><span id="line-911"> currentRegionBoundariesInformation.metaFirstKey = null;</span>
<span class="source-line-no">912</span><span id="line-912"> if (currentRegionBoundariesInformation.metaLastKey.length == 0)</span>
<span class="source-line-no">913</span><span id="line-913"> currentRegionBoundariesInformation.metaLastKey = null;</span>
<span class="source-line-no">914</span><span id="line-914"></span>
<span class="source-line-no">915</span><span id="line-915"> // For a region to be correct, we need the META start key to be smaller or equal to the</span>
<span class="source-line-no">916</span><span id="line-916"> // smallest start key from all the stores, and the start key from the next META entry to</span>
<span class="source-line-no">917</span><span id="line-917"> // be bigger than the last key from all the current stores. First region start key is null;</span>
<span class="source-line-no">918</span><span id="line-918"> // Last region end key is null; some regions can be empty and not have any store.</span>
<span class="source-line-no">919</span><span id="line-919"></span>
<span class="source-line-no">920</span><span id="line-920"> boolean valid = true;</span>
<span class="source-line-no">921</span><span id="line-921"> // Checking start key.</span>
<span class="source-line-no">922</span><span id="line-922"> if (</span>
<span class="source-line-no">923</span><span id="line-923"> (currentRegionBoundariesInformation.storesFirstKey != null)</span>
<span class="source-line-no">924</span><span id="line-924"> &amp;&amp; (currentRegionBoundariesInformation.metaFirstKey != null)</span>
<span class="source-line-no">925</span><span id="line-925"> ) {</span>
<span class="source-line-no">926</span><span id="line-926"> valid = valid &amp;&amp; comparator.compare(currentRegionBoundariesInformation.storesFirstKey,</span>
<span class="source-line-no">927</span><span id="line-927"> currentRegionBoundariesInformation.metaFirstKey) &gt;= 0;</span>
<span class="source-line-no">928</span><span id="line-928"> }</span>
<span class="source-line-no">929</span><span id="line-929"> // Checking stop key.</span>
<span class="source-line-no">930</span><span id="line-930"> if (</span>
<span class="source-line-no">931</span><span id="line-931"> (currentRegionBoundariesInformation.storesLastKey != null)</span>
<span class="source-line-no">932</span><span id="line-932"> &amp;&amp; (currentRegionBoundariesInformation.metaLastKey != null)</span>
<span class="source-line-no">933</span><span id="line-933"> ) {</span>
<span class="source-line-no">934</span><span id="line-934"> valid = valid &amp;&amp; comparator.compare(currentRegionBoundariesInformation.storesLastKey,</span>
<span class="source-line-no">935</span><span id="line-935"> currentRegionBoundariesInformation.metaLastKey) &lt; 0;</span>
<span class="source-line-no">936</span><span id="line-936"> }</span>
<span class="source-line-no">937</span><span id="line-937"> if (!valid) {</span>
<span class="source-line-no">938</span><span id="line-938"> errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries",</span>
<span class="source-line-no">939</span><span id="line-939"> tablesInfo.get(regionInfo.getTable()));</span>
<span class="source-line-no">940</span><span id="line-940"> LOG.warn("Region's boundaries not aligned between stores and META for:");</span>
<span class="source-line-no">941</span><span id="line-941"> LOG.warn(Objects.toString(currentRegionBoundariesInformation));</span>
<span class="source-line-no">942</span><span id="line-942"> }</span>
<span class="source-line-no">943</span><span id="line-943"> }</span>
<span class="source-line-no">944</span><span id="line-944"> } catch (IOException e) {</span>
<span class="source-line-no">945</span><span id="line-945"> LOG.error(e.toString(), e);</span>
<span class="source-line-no">946</span><span id="line-946"> }</span>
<span class="source-line-no">947</span><span id="line-947"> }</span>
<span class="source-line-no">948</span><span id="line-948"></span>
<span class="source-line-no">949</span><span id="line-949"> /**</span>
<span class="source-line-no">950</span><span id="line-950"> * Iterates through the list of all orphan/invalid regiondirs.</span>
<span class="source-line-no">951</span><span id="line-951"> */</span>
<span class="source-line-no">952</span><span id="line-952"> private void adoptHdfsOrphans(Collection&lt;HbckRegionInfo&gt; orphanHdfsDirs) throws IOException {</span>
<span class="source-line-no">953</span><span id="line-953"> for (HbckRegionInfo hi : orphanHdfsDirs) {</span>
<span class="source-line-no">954</span><span id="line-954"> LOG.info("Attempting to handle orphan hdfs dir: " + hi.getHdfsRegionDir());</span>
<span class="source-line-no">955</span><span id="line-955"> adoptHdfsOrphan(hi);</span>
<span class="source-line-no">956</span><span id="line-956"> }</span>
<span class="source-line-no">957</span><span id="line-957"> }</span>
<span class="source-line-no">958</span><span id="line-958"></span>
<span class="source-line-no">959</span><span id="line-959"> /**</span>
<span class="source-line-no">960</span><span id="line-960"> * Orphaned regions are regions without a .regioninfo file in them. We "adopt" these orphans by</span>
<span class="source-line-no">961</span><span id="line-961"> * creating a new region, and moving the column families, recovered edits, WALs, into the new</span>
<span class="source-line-no">962</span><span id="line-962"> * region dir. We determine the region startkey and endkeys by looking at all of the hfiles inside</span>
<span class="source-line-no">963</span><span id="line-963"> * the column families to identify the min and max keys. The resulting region will likely violate</span>
<span class="source-line-no">964</span><span id="line-964"> * table integrity but will be dealt with by merging overlapping regions.</span>
<span class="source-line-no">965</span><span id="line-965"> */</span>
<span class="source-line-no">966</span><span id="line-966"> @SuppressWarnings("deprecation")</span>
<span class="source-line-no">967</span><span id="line-967"> private void adoptHdfsOrphan(HbckRegionInfo hi) throws IOException {</span>
<span class="source-line-no">968</span><span id="line-968"> Path p = hi.getHdfsRegionDir();</span>
<span class="source-line-no">969</span><span id="line-969"> FileSystem fs = p.getFileSystem(getConf());</span>
<span class="source-line-no">970</span><span id="line-970"> FileStatus[] dirs = fs.listStatus(p);</span>
<span class="source-line-no">971</span><span id="line-971"> if (dirs == null) {</span>
<span class="source-line-no">972</span><span id="line-972"> LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " + p</span>
<span class="source-line-no">973</span><span id="line-973"> + ". This dir could probably be deleted.");</span>
<span class="source-line-no">974</span><span id="line-974"> return;</span>
<span class="source-line-no">975</span><span id="line-975"> }</span>
<span class="source-line-no">976</span><span id="line-976"></span>
<span class="source-line-no">977</span><span id="line-977"> TableName tableName = hi.getTableName();</span>
<span class="source-line-no">978</span><span id="line-978"> HbckTableInfo tableInfo = tablesInfo.get(tableName);</span>
<span class="source-line-no">979</span><span id="line-979"> Preconditions.checkNotNull(tableInfo, "Table '" + tableName + "' not present!");</span>
<span class="source-line-no">980</span><span id="line-980"> TableDescriptor template = tableInfo.getTableDescriptor();</span>
<span class="source-line-no">981</span><span id="line-981"></span>
<span class="source-line-no">982</span><span id="line-982"> // find min and max key values</span>
<span class="source-line-no">983</span><span id="line-983"> Pair&lt;byte[], byte[]&gt; orphanRegionRange = null;</span>
<span class="source-line-no">984</span><span id="line-984"> for (FileStatus cf : dirs) {</span>
<span class="source-line-no">985</span><span id="line-985"> String cfName = cf.getPath().getName();</span>
<span class="source-line-no">986</span><span id="line-986"> // TODO Figure out what the special dirs are</span>
<span class="source-line-no">987</span><span id="line-987"> if (cfName.startsWith(".") || cfName.equals(HConstants.SPLIT_LOGDIR_NAME)) continue;</span>
<span class="source-line-no">988</span><span id="line-988"></span>
<span class="source-line-no">989</span><span id="line-989"> FileStatus[] hfiles = fs.listStatus(cf.getPath());</span>
<span class="source-line-no">990</span><span id="line-990"> for (FileStatus hfile : hfiles) {</span>
<span class="source-line-no">991</span><span id="line-991"> byte[] start, end;</span>
<span class="source-line-no">992</span><span id="line-992"> HFile.Reader hf = null;</span>
<span class="source-line-no">993</span><span id="line-993"> try {</span>
<span class="source-line-no">994</span><span id="line-994"> hf = HFile.createReader(fs, hfile.getPath(), CacheConfig.DISABLED, true, getConf());</span>
<span class="source-line-no">995</span><span id="line-995"> Optional&lt;ExtendedCell&gt; startKv = hf.getFirstKey();</span>
<span class="source-line-no">996</span><span id="line-996"> start = CellUtil.cloneRow(startKv.get());</span>
<span class="source-line-no">997</span><span id="line-997"> Optional&lt;ExtendedCell&gt; endKv = hf.getLastKey();</span>
<span class="source-line-no">998</span><span id="line-998"> end = CellUtil.cloneRow(endKv.get());</span>
<span class="source-line-no">999</span><span id="line-999"> } catch (Exception ioe) {</span>
<span class="source-line-no">1000</span><span id="line-1000"> LOG.warn("Problem reading orphan file " + hfile + ", skipping");</span>
<span class="source-line-no">1001</span><span id="line-1001"> continue;</span>
<span class="source-line-no">1002</span><span id="line-1002"> } finally {</span>
<span class="source-line-no">1003</span><span id="line-1003"> if (hf != null) {</span>
<span class="source-line-no">1004</span><span id="line-1004"> hf.close();</span>
<span class="source-line-no">1005</span><span id="line-1005"> }</span>
<span class="source-line-no">1006</span><span id="line-1006"> }</span>
<span class="source-line-no">1007</span><span id="line-1007"></span>
<span class="source-line-no">1008</span><span id="line-1008"> // expand the range to include the range of all hfiles</span>
<span class="source-line-no">1009</span><span id="line-1009"> if (orphanRegionRange == null) {</span>
<span class="source-line-no">1010</span><span id="line-1010"> // first range</span>
<span class="source-line-no">1011</span><span id="line-1011"> orphanRegionRange = new Pair&lt;&gt;(start, end);</span>
<span class="source-line-no">1012</span><span id="line-1012"> } else {</span>
<span class="source-line-no">1013</span><span id="line-1013"> // TODO add test</span>
<span class="source-line-no">1014</span><span id="line-1014"></span>
<span class="source-line-no">1015</span><span id="line-1015"> // expand range only if the hfile is wider.</span>
<span class="source-line-no">1016</span><span id="line-1016"> if (Bytes.compareTo(orphanRegionRange.getFirst(), start) &gt; 0) {</span>
<span class="source-line-no">1017</span><span id="line-1017"> orphanRegionRange.setFirst(start);</span>
<span class="source-line-no">1018</span><span id="line-1018"> }</span>
<span class="source-line-no">1019</span><span id="line-1019"> if (Bytes.compareTo(orphanRegionRange.getSecond(), end) &lt; 0) {</span>
<span class="source-line-no">1020</span><span id="line-1020"> orphanRegionRange.setSecond(end);</span>
<span class="source-line-no">1021</span><span id="line-1021"> }</span>
<span class="source-line-no">1022</span><span id="line-1022"> }</span>
<span class="source-line-no">1023</span><span id="line-1023"> }</span>
<span class="source-line-no">1024</span><span id="line-1024"> }</span>
<span class="source-line-no">1025</span><span id="line-1025"> if (orphanRegionRange == null) {</span>
<span class="source-line-no">1026</span><span id="line-1026"> LOG.warn("No data in dir " + p + ", sidelining data");</span>
<span class="source-line-no">1027</span><span id="line-1027"> fixes++;</span>
<span class="source-line-no">1028</span><span id="line-1028"> sidelineRegionDir(fs, hi);</span>
<span class="source-line-no">1029</span><span id="line-1029"> return;</span>
<span class="source-line-no">1030</span><span id="line-1030"> }</span>
<span class="source-line-no">1031</span><span id="line-1031"> LOG.info("Min max keys are : [" + Bytes.toString(orphanRegionRange.getFirst()) + ", "</span>
<span class="source-line-no">1032</span><span id="line-1032"> + Bytes.toString(orphanRegionRange.getSecond()) + ")");</span>
<span class="source-line-no">1033</span><span id="line-1033"></span>
<span class="source-line-no">1034</span><span id="line-1034"> // create new region on hdfs. move data into place.</span>
<span class="source-line-no">1035</span><span id="line-1035"> RegionInfo regionInfo = RegionInfoBuilder.newBuilder(template.getTableName())</span>
<span class="source-line-no">1036</span><span id="line-1036"> .setStartKey(orphanRegionRange.getFirst())</span>
<span class="source-line-no">1037</span><span id="line-1037"> .setEndKey(Bytes.add(orphanRegionRange.getSecond(), new byte[1])).build();</span>
<span class="source-line-no">1038</span><span id="line-1038"> LOG.info("Creating new region : " + regionInfo);</span>
<span class="source-line-no">1039</span><span id="line-1039"> HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), regionInfo, template);</span>
<span class="source-line-no">1040</span><span id="line-1040"> Path target = region.getRegionFileSystem().getRegionDir();</span>
<span class="source-line-no">1041</span><span id="line-1041"></span>
<span class="source-line-no">1042</span><span id="line-1042"> // rename all the data to new region</span>
<span class="source-line-no">1043</span><span id="line-1043"> mergeRegionDirs(target, hi);</span>
<span class="source-line-no">1044</span><span id="line-1044"> fixes++;</span>
<span class="source-line-no">1045</span><span id="line-1045"> }</span>
<span class="source-line-no">1046</span><span id="line-1046"></span>
<span class="source-line-no">1047</span><span id="line-1047"> /**</span>
<span class="source-line-no">1048</span><span id="line-1048"> * This method determines if there are table integrity errors in HDFS. If there are errors and the</span>
<span class="source-line-no">1049</span><span id="line-1049"> * appropriate "fix" options are enabled, the method will first correct orphan regions making them</span>
<span class="source-line-no">1050</span><span id="line-1050"> * into legit regiondirs, and then reload to merge potentially overlapping regions.</span>
<span class="source-line-no">1051</span><span id="line-1051"> * @return number of table integrity errors found</span>
<span class="source-line-no">1052</span><span id="line-1052"> */</span>
<span class="source-line-no">1053</span><span id="line-1053"> private int restoreHdfsIntegrity() throws IOException, InterruptedException {</span>
<span class="source-line-no">1054</span><span id="line-1054"> // Determine what's on HDFS</span>
<span class="source-line-no">1055</span><span id="line-1055"> LOG.info("Loading HBase regioninfo from HDFS...");</span>
<span class="source-line-no">1056</span><span id="line-1056"> loadHdfsRegionDirs(); // populating regioninfo table.</span>
<span class="source-line-no">1057</span><span id="line-1057"></span>
<span class="source-line-no">1058</span><span id="line-1058"> int errs = errors.getErrorList().size();</span>
<span class="source-line-no">1059</span><span id="line-1059"> // First time just get suggestions.</span>
<span class="source-line-no">1060</span><span id="line-1060"> tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.</span>
<span class="source-line-no">1061</span><span id="line-1061"> checkHdfsIntegrity(false, false);</span>
<span class="source-line-no">1062</span><span id="line-1062"></span>
<span class="source-line-no">1063</span><span id="line-1063"> if (errors.getErrorList().size() == errs) {</span>
<span class="source-line-no">1064</span><span id="line-1064"> LOG.info("No integrity errors. We are done with this phase. Glorious.");</span>
<span class="source-line-no">1065</span><span id="line-1065"> return 0;</span>
<span class="source-line-no">1066</span><span id="line-1066"> }</span>
<span class="source-line-no">1067</span><span id="line-1067"></span>
<span class="source-line-no">1068</span><span id="line-1068"> if (shouldFixHdfsOrphans() &amp;&amp; orphanHdfsDirs.size() &gt; 0) {</span>
<span class="source-line-no">1069</span><span id="line-1069"> adoptHdfsOrphans(orphanHdfsDirs);</span>
<span class="source-line-no">1070</span><span id="line-1070"> // TODO optimize by incrementally adding instead of reloading.</span>
<span class="source-line-no">1071</span><span id="line-1071"> }</span>
<span class="source-line-no">1072</span><span id="line-1072"></span>
<span class="source-line-no">1073</span><span id="line-1073"> // Make sure there are no holes now.</span>
<span class="source-line-no">1074</span><span id="line-1074"> if (shouldFixHdfsHoles()) {</span>
<span class="source-line-no">1075</span><span id="line-1075"> clearState(); // this also resets # fixes.</span>
<span class="source-line-no">1076</span><span id="line-1076"> loadHdfsRegionDirs();</span>
<span class="source-line-no">1077</span><span id="line-1077"> tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.</span>
<span class="source-line-no">1078</span><span id="line-1078"> tablesInfo = checkHdfsIntegrity(shouldFixHdfsHoles(), false);</span>
<span class="source-line-no">1079</span><span id="line-1079"> }</span>
<span class="source-line-no">1080</span><span id="line-1080"></span>
<span class="source-line-no">1081</span><span id="line-1081"> // Now we fix overlaps</span>
<span class="source-line-no">1082</span><span id="line-1082"> if (shouldFixHdfsOverlaps()) {</span>
<span class="source-line-no">1083</span><span id="line-1083"> // second pass we fix overlaps.</span>
<span class="source-line-no">1084</span><span id="line-1084"> clearState(); // this also resets # fixes.</span>
<span class="source-line-no">1085</span><span id="line-1085"> loadHdfsRegionDirs();</span>
<span class="source-line-no">1086</span><span id="line-1086"> tablesInfo = loadHdfsRegionInfos(); // update tableInfos based on region info in fs.</span>
<span class="source-line-no">1087</span><span id="line-1087"> tablesInfo = checkHdfsIntegrity(false, shouldFixHdfsOverlaps());</span>
<span class="source-line-no">1088</span><span id="line-1088"> }</span>
<span class="source-line-no">1089</span><span id="line-1089"></span>
<span class="source-line-no">1090</span><span id="line-1090"> return errors.getErrorList().size();</span>
<span class="source-line-no">1091</span><span id="line-1091"> }</span>
<span class="source-line-no">1092</span><span id="line-1092"></span>
<span class="source-line-no">1093</span><span id="line-1093"> /**</span>
<span class="source-line-no">1094</span><span id="line-1094"> * Scan all the store file names to find any lingering reference files, which refer to some</span>
<span class="source-line-no">1095</span><span id="line-1095"> * none-exiting files. If "fix" option is enabled, any lingering reference file will be sidelined</span>
<span class="source-line-no">1096</span><span id="line-1096"> * if found.</span>
<span class="source-line-no">1097</span><span id="line-1097"> * &lt;p&gt;</span>
<span class="source-line-no">1098</span><span id="line-1098"> * Lingering reference file prevents a region from opening. It has to be fixed before a cluster</span>
<span class="source-line-no">1099</span><span id="line-1099"> * can start properly.</span>
<span class="source-line-no">1100</span><span id="line-1100"> */</span>
<span class="source-line-no">1101</span><span id="line-1101"> private void offlineReferenceFileRepair() throws IOException, InterruptedException {</span>
<span class="source-line-no">1102</span><span id="line-1102"> clearState();</span>
<span class="source-line-no">1103</span><span id="line-1103"> Configuration conf = getConf();</span>
<span class="source-line-no">1104</span><span id="line-1104"> Path hbaseRoot = CommonFSUtils.getRootDir(conf);</span>
<span class="source-line-no">1105</span><span id="line-1105"> FileSystem fs = hbaseRoot.getFileSystem(conf);</span>
<span class="source-line-no">1106</span><span id="line-1106"> LOG.info("Computing mapping of all store files");</span>
<span class="source-line-no">1107</span><span id="line-1107"> Map&lt;String, Path&gt; allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot,</span>
<span class="source-line-no">1108</span><span id="line-1108"> new FSUtils.ReferenceFileFilter(fs), executor, errors);</span>
<span class="source-line-no">1109</span><span id="line-1109"> errors.print("");</span>
<span class="source-line-no">1110</span><span id="line-1110"> LOG.info("Validating mapping using HDFS state");</span>
<span class="source-line-no">1111</span><span id="line-1111"> for (Path path : allFiles.values()) {</span>
<span class="source-line-no">1112</span><span id="line-1112"> Path referredToFile = StoreFileInfo.getReferredToFile(path);</span>
<span class="source-line-no">1113</span><span id="line-1113"> if (fs.exists(referredToFile)) continue; // good, expected</span>
<span class="source-line-no">1114</span><span id="line-1114"></span>
<span class="source-line-no">1115</span><span id="line-1115"> // Found a lingering reference file</span>
<span class="source-line-no">1116</span><span id="line-1116"> errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,</span>
<span class="source-line-no">1117</span><span id="line-1117"> "Found lingering reference file " + path);</span>
<span class="source-line-no">1118</span><span id="line-1118"> if (!shouldFixReferenceFiles()) continue;</span>
<span class="source-line-no">1119</span><span id="line-1119"></span>
<span class="source-line-no">1120</span><span id="line-1120"> // Now, trying to fix it since requested</span>
<span class="source-line-no">1121</span><span id="line-1121"> boolean success = false;</span>
<span class="source-line-no">1122</span><span id="line-1122"> String pathStr = path.toString();</span>
<span class="source-line-no">1123</span><span id="line-1123"></span>
<span class="source-line-no">1124</span><span id="line-1124"> // A reference file path should be like</span>
<span class="source-line-no">1125</span><span id="line-1125"> // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/referred_file.region_name</span>
<span class="source-line-no">1126</span><span id="line-1126"> // Up 5 directories to get the root folder.</span>
<span class="source-line-no">1127</span><span id="line-1127"> // So the file will be sidelined to a similar folder structure.</span>
<span class="source-line-no">1128</span><span id="line-1128"> int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);</span>
<span class="source-line-no">1129</span><span id="line-1129"> for (int i = 0; index &gt; 0 &amp;&amp; i &lt; 5; i++) {</span>
<span class="source-line-no">1130</span><span id="line-1130"> index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index - 1);</span>
<span class="source-line-no">1131</span><span id="line-1131"> }</span>
<span class="source-line-no">1132</span><span id="line-1132"> if (index &gt; 0) {</span>
<span class="source-line-no">1133</span><span id="line-1133"> Path rootDir = getSidelineDir();</span>
<span class="source-line-no">1134</span><span id="line-1134"> Path dst = new Path(rootDir, pathStr.substring(index + 1));</span>
<span class="source-line-no">1135</span><span id="line-1135"> fs.mkdirs(dst.getParent());</span>
<span class="source-line-no">1136</span><span id="line-1136"> LOG.info("Trying to sideline reference file " + path + " to " + dst);</span>
<span class="source-line-no">1137</span><span id="line-1137"> setShouldRerun();</span>
<span class="source-line-no">1138</span><span id="line-1138"></span>
<span class="source-line-no">1139</span><span id="line-1139"> success = fs.rename(path, dst);</span>
<span class="source-line-no">1140</span><span id="line-1140"> debugLsr(dst);</span>
<span class="source-line-no">1141</span><span id="line-1141"></span>
<span class="source-line-no">1142</span><span id="line-1142"> }</span>
<span class="source-line-no">1143</span><span id="line-1143"> if (!success) {</span>
<span class="source-line-no">1144</span><span id="line-1144"> LOG.error("Failed to sideline reference file " + path);</span>
<span class="source-line-no">1145</span><span id="line-1145"> }</span>
<span class="source-line-no">1146</span><span id="line-1146"> }</span>
<span class="source-line-no">1147</span><span id="line-1147"> }</span>
<span class="source-line-no">1148</span><span id="line-1148"></span>
<span class="source-line-no">1149</span><span id="line-1149"> /**</span>
<span class="source-line-no">1150</span><span id="line-1150"> * Scan all the store file names to find any lingering HFileLink files, which refer to some</span>
<span class="source-line-no">1151</span><span id="line-1151"> * none-exiting files. If "fix" option is enabled, any lingering HFileLink file will be sidelined</span>
<span class="source-line-no">1152</span><span id="line-1152"> * if found.</span>
<span class="source-line-no">1153</span><span id="line-1153"> */</span>
<span class="source-line-no">1154</span><span id="line-1154"> private void offlineHLinkFileRepair() throws IOException, InterruptedException {</span>
<span class="source-line-no">1155</span><span id="line-1155"> Configuration conf = getConf();</span>
<span class="source-line-no">1156</span><span id="line-1156"> Path hbaseRoot = CommonFSUtils.getRootDir(conf);</span>
<span class="source-line-no">1157</span><span id="line-1157"> FileSystem fs = hbaseRoot.getFileSystem(conf);</span>
<span class="source-line-no">1158</span><span id="line-1158"> LOG.info("Computing mapping of all link files");</span>
<span class="source-line-no">1159</span><span id="line-1159"> Map&lt;String, Path&gt; allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot,</span>
<span class="source-line-no">1160</span><span id="line-1160"> new FSUtils.HFileLinkFilter(), executor, errors);</span>
<span class="source-line-no">1161</span><span id="line-1161"> errors.print("");</span>
<span class="source-line-no">1162</span><span id="line-1162"></span>
<span class="source-line-no">1163</span><span id="line-1163"> LOG.info("Validating mapping using HDFS state");</span>
<span class="source-line-no">1164</span><span id="line-1164"> for (Path path : allFiles.values()) {</span>
<span class="source-line-no">1165</span><span id="line-1165"> // building HFileLink object to gather locations</span>
<span class="source-line-no">1166</span><span id="line-1166"> HFileLink actualLink = HFileLink.buildFromHFileLinkPattern(conf, path);</span>
<span class="source-line-no">1167</span><span id="line-1167"> if (actualLink.exists(fs)) continue; // good, expected</span>
<span class="source-line-no">1168</span><span id="line-1168"></span>
<span class="source-line-no">1169</span><span id="line-1169"> // Found a lingering HFileLink</span>
<span class="source-line-no">1170</span><span id="line-1170"> errors.reportError(ERROR_CODE.LINGERING_HFILELINK, "Found lingering HFileLink " + path);</span>
<span class="source-line-no">1171</span><span id="line-1171"> if (!shouldFixHFileLinks()) continue;</span>
<span class="source-line-no">1172</span><span id="line-1172"></span>
<span class="source-line-no">1173</span><span id="line-1173"> // Now, trying to fix it since requested</span>
<span class="source-line-no">1174</span><span id="line-1174"> setShouldRerun();</span>
<span class="source-line-no">1175</span><span id="line-1175"></span>
<span class="source-line-no">1176</span><span id="line-1176"> // An HFileLink path should be like</span>
<span class="source-line-no">1177</span><span id="line-1177"> // ${hbase.rootdir}/data/namespace/table_name/region_id/family_name/linkedtable=linkedregionname-linkedhfilename</span>
<span class="source-line-no">1178</span><span id="line-1178"> // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same</span>
<span class="source-line-no">1179</span><span id="line-1179"> // folder structure.</span>
<span class="source-line-no">1180</span><span id="line-1180"> boolean success = sidelineFile(fs, hbaseRoot, path);</span>
<span class="source-line-no">1181</span><span id="line-1181"></span>
<span class="source-line-no">1182</span><span id="line-1182"> if (!success) {</span>
<span class="source-line-no">1183</span><span id="line-1183"> LOG.error("Failed to sideline HFileLink file " + path);</span>
<span class="source-line-no">1184</span><span id="line-1184"> }</span>
<span class="source-line-no">1185</span><span id="line-1185"></span>
<span class="source-line-no">1186</span><span id="line-1186"> // An HFileLink backreference path should be like</span>
<span class="source-line-no">1187</span><span id="line-1187"> // ${hbase.rootdir}/archive/data/namespace/table_name/region_id/family_name/.links-linkedhfilename</span>
<span class="source-line-no">1188</span><span id="line-1188"> // sidelineing will happen in the ${hbase.rootdir}/${sidelinedir} directory with the same</span>
<span class="source-line-no">1189</span><span id="line-1189"> // folder structure.</span>
<span class="source-line-no">1190</span><span id="line-1190"> Path backRefPath = FileLink.getBackReferencesDir(</span>
<span class="source-line-no">1191</span><span id="line-1191"> HFileArchiveUtil.getStoreArchivePath(conf,</span>
<span class="source-line-no">1192</span><span id="line-1192"> HFileLink.getReferencedTableName(path.getName().toString()),</span>
<span class="source-line-no">1193</span><span id="line-1193"> HFileLink.getReferencedRegionName(path.getName().toString()), path.getParent().getName()),</span>
<span class="source-line-no">1194</span><span id="line-1194"> HFileLink.getReferencedHFileName(path.getName().toString()));</span>
<span class="source-line-no">1195</span><span id="line-1195"> success = sidelineFile(fs, hbaseRoot, backRefPath);</span>
<span class="source-line-no">1196</span><span id="line-1196"></span>
<span class="source-line-no">1197</span><span id="line-1197"> if (!success) {</span>
<span class="source-line-no">1198</span><span id="line-1198"> LOG.error("Failed to sideline HFileLink backreference file " + path);</span>
<span class="source-line-no">1199</span><span id="line-1199"> }</span>
<span class="source-line-no">1200</span><span id="line-1200"> }</span>
<span class="source-line-no">1201</span><span id="line-1201"> }</span>
<span class="source-line-no">1202</span><span id="line-1202"></span>
<span class="source-line-no">1203</span><span id="line-1203"> private boolean sidelineFile(FileSystem fs, Path hbaseRoot, Path path) throws IOException {</span>
<span class="source-line-no">1204</span><span id="line-1204"> URI uri = hbaseRoot.toUri().relativize(path.toUri());</span>
<span class="source-line-no">1205</span><span id="line-1205"> if (uri.isAbsolute()) return false;</span>
<span class="source-line-no">1206</span><span id="line-1206"> String relativePath = uri.getPath();</span>
<span class="source-line-no">1207</span><span id="line-1207"> Path rootDir = getSidelineDir();</span>
<span class="source-line-no">1208</span><span id="line-1208"> Path dst = new Path(rootDir, relativePath);</span>
<span class="source-line-no">1209</span><span id="line-1209"> boolean pathCreated = fs.mkdirs(dst.getParent());</span>
<span class="source-line-no">1210</span><span id="line-1210"> if (!pathCreated) {</span>
<span class="source-line-no">1211</span><span id="line-1211"> LOG.error("Failed to create path: " + dst.getParent());</span>
<span class="source-line-no">1212</span><span id="line-1212"> return false;</span>
<span class="source-line-no">1213</span><span id="line-1213"> }</span>
<span class="source-line-no">1214</span><span id="line-1214"> LOG.info("Trying to sideline file " + path + " to " + dst);</span>
<span class="source-line-no">1215</span><span id="line-1215"> return fs.rename(path, dst);</span>
<span class="source-line-no">1216</span><span id="line-1216"> }</span>
<span class="source-line-no">1217</span><span id="line-1217"></span>
<span class="source-line-no">1218</span><span id="line-1218"> /**</span>
<span class="source-line-no">1219</span><span id="line-1219"> * TODO -- need to add tests for this.</span>
<span class="source-line-no">1220</span><span id="line-1220"> */</span>
<span class="source-line-no">1221</span><span id="line-1221"> private void reportEmptyMetaCells() {</span>
<span class="source-line-no">1222</span><span id="line-1222"> errors.print("Number of empty REGIONINFO_QUALIFIER rows in hbase:meta: "</span>
<span class="source-line-no">1223</span><span id="line-1223"> + emptyRegionInfoQualifiers.size());</span>
<span class="source-line-no">1224</span><span id="line-1224"> if (details) {</span>
<span class="source-line-no">1225</span><span id="line-1225"> for (Result r : emptyRegionInfoQualifiers) {</span>
<span class="source-line-no">1226</span><span id="line-1226"> errors.print(" " + r);</span>
<span class="source-line-no">1227</span><span id="line-1227"> }</span>
<span class="source-line-no">1228</span><span id="line-1228"> }</span>
<span class="source-line-no">1229</span><span id="line-1229"> }</span>
<span class="source-line-no">1230</span><span id="line-1230"></span>
<span class="source-line-no">1231</span><span id="line-1231"> /**</span>
<span class="source-line-no">1232</span><span id="line-1232"> * TODO -- need to add tests for this.</span>
<span class="source-line-no">1233</span><span id="line-1233"> */</span>
<span class="source-line-no">1234</span><span id="line-1234"> private void reportTablesInFlux() {</span>
<span class="source-line-no">1235</span><span id="line-1235"> AtomicInteger numSkipped = new AtomicInteger(0);</span>
<span class="source-line-no">1236</span><span id="line-1236"> TableDescriptor[] allTables = getTables(numSkipped);</span>
<span class="source-line-no">1237</span><span id="line-1237"> errors.print("Number of Tables: " + allTables.length);</span>
<span class="source-line-no">1238</span><span id="line-1238"> if (details) {</span>
<span class="source-line-no">1239</span><span id="line-1239"> if (numSkipped.get() &gt; 0) {</span>
<span class="source-line-no">1240</span><span id="line-1240"> errors.detail("Number of Tables in flux: " + numSkipped.get());</span>
<span class="source-line-no">1241</span><span id="line-1241"> }</span>
<span class="source-line-no">1242</span><span id="line-1242"> for (TableDescriptor td : allTables) {</span>
<span class="source-line-no">1243</span><span id="line-1243"> errors.detail(" Table: " + td.getTableName() + "\t" + (td.isReadOnly() ? "ro" : "rw")</span>
<span class="source-line-no">1244</span><span id="line-1244"> + "\t" + (td.isMetaRegion() ? "META" : " ") + "\t" + " families: "</span>
<span class="source-line-no">1245</span><span id="line-1245"> + td.getColumnFamilyCount());</span>
<span class="source-line-no">1246</span><span id="line-1246"> }</span>
<span class="source-line-no">1247</span><span id="line-1247"> }</span>
<span class="source-line-no">1248</span><span id="line-1248"> }</span>
<span class="source-line-no">1249</span><span id="line-1249"></span>
<span class="source-line-no">1250</span><span id="line-1250"> public HbckErrorReporter getErrors() {</span>
<span class="source-line-no">1251</span><span id="line-1251"> return errors;</span>
<span class="source-line-no">1252</span><span id="line-1252"> }</span>
<span class="source-line-no">1253</span><span id="line-1253"></span>
<span class="source-line-no">1254</span><span id="line-1254"> /**</span>
<span class="source-line-no">1255</span><span id="line-1255"> * Populate hbi's from regionInfos loaded from file system.</span>
<span class="source-line-no">1256</span><span id="line-1256"> */</span>
<span class="source-line-no">1257</span><span id="line-1257"> private SortedMap&lt;TableName, HbckTableInfo&gt; loadHdfsRegionInfos()</span>
<span class="source-line-no">1258</span><span id="line-1258"> throws IOException, InterruptedException {</span>
<span class="source-line-no">1259</span><span id="line-1259"> tablesInfo.clear(); // regenerating the data</span>
<span class="source-line-no">1260</span><span id="line-1260"> // generate region split structure</span>
<span class="source-line-no">1261</span><span id="line-1261"> Collection&lt;HbckRegionInfo&gt; hbckRegionInfos = regionInfoMap.values();</span>
<span class="source-line-no">1262</span><span id="line-1262"></span>
<span class="source-line-no">1263</span><span id="line-1263"> // Parallelized read of .regioninfo files.</span>
<span class="source-line-no">1264</span><span id="line-1264"> List&lt;WorkItemHdfsRegionInfo&gt; hbis = new ArrayList&lt;&gt;(hbckRegionInfos.size());</span>
<span class="source-line-no">1265</span><span id="line-1265"> List&lt;Future&lt;Void&gt;&gt; hbiFutures;</span>
<span class="source-line-no">1266</span><span id="line-1266"></span>
<span class="source-line-no">1267</span><span id="line-1267"> for (HbckRegionInfo hbi : hbckRegionInfos) {</span>
<span class="source-line-no">1268</span><span id="line-1268"> WorkItemHdfsRegionInfo work = new WorkItemHdfsRegionInfo(hbi, this, errors);</span>
<span class="source-line-no">1269</span><span id="line-1269"> hbis.add(work);</span>
<span class="source-line-no">1270</span><span id="line-1270"> }</span>
<span class="source-line-no">1271</span><span id="line-1271"></span>
<span class="source-line-no">1272</span><span id="line-1272"> // Submit and wait for completion</span>
<span class="source-line-no">1273</span><span id="line-1273"> hbiFutures = executor.invokeAll(hbis);</span>
<span class="source-line-no">1274</span><span id="line-1274"></span>
<span class="source-line-no">1275</span><span id="line-1275"> for (int i = 0; i &lt; hbiFutures.size(); i++) {</span>
<span class="source-line-no">1276</span><span id="line-1276"> WorkItemHdfsRegionInfo work = hbis.get(i);</span>
<span class="source-line-no">1277</span><span id="line-1277"> Future&lt;Void&gt; f = hbiFutures.get(i);</span>
<span class="source-line-no">1278</span><span id="line-1278"> try {</span>
<span class="source-line-no">1279</span><span id="line-1279"> f.get();</span>
<span class="source-line-no">1280</span><span id="line-1280"> } catch (ExecutionException e) {</span>
<span class="source-line-no">1281</span><span id="line-1281"> LOG.warn("Failed to read .regioninfo file for region " + work.hbi.getRegionNameAsString(),</span>
<span class="source-line-no">1282</span><span id="line-1282"> e.getCause());</span>
<span class="source-line-no">1283</span><span id="line-1283"> }</span>
<span class="source-line-no">1284</span><span id="line-1284"> }</span>
<span class="source-line-no">1285</span><span id="line-1285"></span>
<span class="source-line-no">1286</span><span id="line-1286"> Path hbaseRoot = CommonFSUtils.getRootDir(getConf());</span>
<span class="source-line-no">1287</span><span id="line-1287"> FileSystem fs = hbaseRoot.getFileSystem(getConf());</span>
<span class="source-line-no">1288</span><span id="line-1288"> // serialized table info gathering.</span>
<span class="source-line-no">1289</span><span id="line-1289"> for (HbckRegionInfo hbi : hbckRegionInfos) {</span>
<span class="source-line-no">1290</span><span id="line-1290"></span>
<span class="source-line-no">1291</span><span id="line-1291"> if (hbi.getHdfsHRI() == null) {</span>
<span class="source-line-no">1292</span><span id="line-1292"> // was an orphan</span>
<span class="source-line-no">1293</span><span id="line-1293"> continue;</span>
<span class="source-line-no">1294</span><span id="line-1294"> }</span>
<span class="source-line-no">1295</span><span id="line-1295"></span>
<span class="source-line-no">1296</span><span id="line-1296"> // get table name from hdfs, populate various HBaseFsck tables.</span>
<span class="source-line-no">1297</span><span id="line-1297"> TableName tableName = hbi.getTableName();</span>
<span class="source-line-no">1298</span><span id="line-1298"> if (tableName == null) {</span>
<span class="source-line-no">1299</span><span id="line-1299"> // There was an entry in hbase:meta not in the HDFS?</span>
<span class="source-line-no">1300</span><span id="line-1300"> LOG.warn("tableName was null for: " + hbi);</span>
<span class="source-line-no">1301</span><span id="line-1301"> continue;</span>
<span class="source-line-no">1302</span><span id="line-1302"> }</span>
<span class="source-line-no">1303</span><span id="line-1303"></span>
<span class="source-line-no">1304</span><span id="line-1304"> HbckTableInfo modTInfo = tablesInfo.get(tableName);</span>
<span class="source-line-no">1305</span><span id="line-1305"> if (modTInfo == null) {</span>
<span class="source-line-no">1306</span><span id="line-1306"> // only executed once per table.</span>
<span class="source-line-no">1307</span><span id="line-1307"> modTInfo = new HbckTableInfo(tableName, this);</span>
<span class="source-line-no">1308</span><span id="line-1308"> tablesInfo.put(tableName, modTInfo);</span>
<span class="source-line-no">1309</span><span id="line-1309"> try {</span>
<span class="source-line-no">1310</span><span id="line-1310"> TableDescriptor htd =</span>
<span class="source-line-no">1311</span><span id="line-1311"> FSTableDescriptors.getTableDescriptorFromFs(fs, hbaseRoot, tableName);</span>
<span class="source-line-no">1312</span><span id="line-1312"> modTInfo.htds.add(htd);</span>
<span class="source-line-no">1313</span><span id="line-1313"> } catch (IOException ioe) {</span>
<span class="source-line-no">1314</span><span id="line-1314"> if (!orphanTableDirs.containsKey(tableName)) {</span>
<span class="source-line-no">1315</span><span id="line-1315"> LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe);</span>
<span class="source-line-no">1316</span><span id="line-1316"> // should only report once for each table</span>
<span class="source-line-no">1317</span><span id="line-1317"> errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE,</span>
<span class="source-line-no">1318</span><span id="line-1318"> "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName);</span>
<span class="source-line-no">1319</span><span id="line-1319"> Set&lt;String&gt; columns = new HashSet&lt;&gt;();</span>
<span class="source-line-no">1320</span><span id="line-1320"> orphanTableDirs.put(tableName, getColumnFamilyList(columns, hbi));</span>
<span class="source-line-no">1321</span><span id="line-1321"> }</span>
<span class="source-line-no">1322</span><span id="line-1322"> }</span>
<span class="source-line-no">1323</span><span id="line-1323"> }</span>
<span class="source-line-no">1324</span><span id="line-1324"> if (!hbi.isSkipChecks()) {</span>
<span class="source-line-no">1325</span><span id="line-1325"> modTInfo.addRegionInfo(hbi);</span>
<span class="source-line-no">1326</span><span id="line-1326"> }</span>
<span class="source-line-no">1327</span><span id="line-1327"> }</span>
<span class="source-line-no">1328</span><span id="line-1328"></span>
<span class="source-line-no">1329</span><span id="line-1329"> loadTableInfosForTablesWithNoRegion();</span>
<span class="source-line-no">1330</span><span id="line-1330"> errors.print("");</span>
<span class="source-line-no">1331</span><span id="line-1331"></span>
<span class="source-line-no">1332</span><span id="line-1332"> return tablesInfo;</span>
<span class="source-line-no">1333</span><span id="line-1333"> }</span>
<span class="source-line-no">1334</span><span id="line-1334"></span>
<span class="source-line-no">1335</span><span id="line-1335"> /**</span>
<span class="source-line-no">1336</span><span id="line-1336"> * To get the column family list according to the column family dirs</span>
<span class="source-line-no">1337</span><span id="line-1337"> * @return a set of column families</span>
<span class="source-line-no">1338</span><span id="line-1338"> */</span>
<span class="source-line-no">1339</span><span id="line-1339"> private Set&lt;String&gt; getColumnFamilyList(Set&lt;String&gt; columns, HbckRegionInfo hbi)</span>
<span class="source-line-no">1340</span><span id="line-1340"> throws IOException {</span>
<span class="source-line-no">1341</span><span id="line-1341"> Path regionDir = hbi.getHdfsRegionDir();</span>
<span class="source-line-no">1342</span><span id="line-1342"> FileSystem fs = regionDir.getFileSystem(getConf());</span>
<span class="source-line-no">1343</span><span id="line-1343"> FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));</span>
<span class="source-line-no">1344</span><span id="line-1344"> for (FileStatus subdir : subDirs) {</span>
<span class="source-line-no">1345</span><span id="line-1345"> String columnfamily = subdir.getPath().getName();</span>
<span class="source-line-no">1346</span><span id="line-1346"> columns.add(columnfamily);</span>
<span class="source-line-no">1347</span><span id="line-1347"> }</span>
<span class="source-line-no">1348</span><span id="line-1348"> return columns;</span>
<span class="source-line-no">1349</span><span id="line-1349"> }</span>
<span class="source-line-no">1350</span><span id="line-1350"></span>
<span class="source-line-no">1351</span><span id="line-1351"> /**</span>
<span class="source-line-no">1352</span><span id="line-1352"> * To fabricate a .tableinfo file with following contents&lt;br&gt;</span>
<span class="source-line-no">1353</span><span id="line-1353"> * 1. the correct tablename &lt;br&gt;</span>
<span class="source-line-no">1354</span><span id="line-1354"> * 2. the correct colfamily list&lt;br&gt;</span>
<span class="source-line-no">1355</span><span id="line-1355"> * 3. the default properties for both {@link TableDescriptor} and</span>
<span class="source-line-no">1356</span><span id="line-1356"> * {@link ColumnFamilyDescriptor}&lt;br&gt;</span>
<span class="source-line-no">1357</span><span id="line-1357"> */</span>
<span class="source-line-no">1358</span><span id="line-1358"> private boolean fabricateTableInfo(FSTableDescriptors fstd, TableName tableName,</span>
<span class="source-line-no">1359</span><span id="line-1359"> Set&lt;String&gt; columns) throws IOException {</span>
<span class="source-line-no">1360</span><span id="line-1360"> if (columns == null || columns.isEmpty()) return false;</span>
<span class="source-line-no">1361</span><span id="line-1361"> TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);</span>
<span class="source-line-no">1362</span><span id="line-1362"> for (String columnfamimly : columns) {</span>
<span class="source-line-no">1363</span><span id="line-1363"> builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnfamimly));</span>
<span class="source-line-no">1364</span><span id="line-1364"> }</span>
<span class="source-line-no">1365</span><span id="line-1365"> fstd.createTableDescriptor(builder.build(), true);</span>
<span class="source-line-no">1366</span><span id="line-1366"> return true;</span>
<span class="source-line-no">1367</span><span id="line-1367"> }</span>
<span class="source-line-no">1368</span><span id="line-1368"></span>
<span class="source-line-no">1369</span><span id="line-1369"> /**</span>
<span class="source-line-no">1370</span><span id="line-1370"> * To fix the empty REGIONINFO_QUALIFIER rows from hbase:meta &lt;br&gt;</span>
<span class="source-line-no">1371</span><span id="line-1371"> */</span>
<span class="source-line-no">1372</span><span id="line-1372"> public void fixEmptyMetaCells() throws IOException {</span>
<span class="source-line-no">1373</span><span id="line-1373"> if (shouldFixEmptyMetaCells() &amp;&amp; !emptyRegionInfoQualifiers.isEmpty()) {</span>
<span class="source-line-no">1374</span><span id="line-1374"> LOG.info("Trying to fix empty REGIONINFO_QUALIFIER hbase:meta rows.");</span>
<span class="source-line-no">1375</span><span id="line-1375"> for (Result region : emptyRegionInfoQualifiers) {</span>
<span class="source-line-no">1376</span><span id="line-1376"> deleteMetaRegion(region.getRow());</span>
<span class="source-line-no">1377</span><span id="line-1377"> errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);</span>
<span class="source-line-no">1378</span><span id="line-1378"> }</span>
<span class="source-line-no">1379</span><span id="line-1379"> emptyRegionInfoQualifiers.clear();</span>
<span class="source-line-no">1380</span><span id="line-1380"> }</span>
<span class="source-line-no">1381</span><span id="line-1381"> }</span>
<span class="source-line-no">1382</span><span id="line-1382"></span>
<span class="source-line-no">1383</span><span id="line-1383"> /**</span>
<span class="source-line-no">1384</span><span id="line-1384"> * To fix orphan table by creating a .tableinfo file under tableDir &lt;br&gt;</span>
<span class="source-line-no">1385</span><span id="line-1385"> * 1. if TableInfo is cached, to recover the .tableinfo accordingly &lt;br&gt;</span>
<span class="source-line-no">1386</span><span id="line-1386"> * 2. else create a default .tableinfo file with following items&lt;br&gt;</span>
<span class="source-line-no">1387</span><span id="line-1387"> * &amp;nbsp;2.1 the correct tablename &lt;br&gt;</span>
<span class="source-line-no">1388</span><span id="line-1388"> * &amp;nbsp;2.2 the correct colfamily list&lt;br&gt;</span>
<span class="source-line-no">1389</span><span id="line-1389"> * &amp;nbsp;2.3 the default properties for both {@link TableDescriptor} and</span>
<span class="source-line-no">1390</span><span id="line-1390"> * {@link ColumnFamilyDescriptor}&lt;br&gt;</span>
<span class="source-line-no">1391</span><span id="line-1391"> */</span>
<span class="source-line-no">1392</span><span id="line-1392"> public void fixOrphanTables() throws IOException {</span>
<span class="source-line-no">1393</span><span id="line-1393"> if (shouldFixTableOrphans() &amp;&amp; !orphanTableDirs.isEmpty()) {</span>
<span class="source-line-no">1394</span><span id="line-1394"></span>
<span class="source-line-no">1395</span><span id="line-1395"> List&lt;TableName&gt; tmpList = new ArrayList&lt;&gt;(orphanTableDirs.keySet().size());</span>
<span class="source-line-no">1396</span><span id="line-1396"> tmpList.addAll(orphanTableDirs.keySet());</span>
<span class="source-line-no">1397</span><span id="line-1397"> TableDescriptor[] htds = getTableDescriptors(tmpList);</span>
<span class="source-line-no">1398</span><span id="line-1398"> Iterator&lt;Entry&lt;TableName, Set&lt;String&gt;&gt;&gt; iter = orphanTableDirs.entrySet().iterator();</span>
<span class="source-line-no">1399</span><span id="line-1399"> int j = 0;</span>
<span class="source-line-no">1400</span><span id="line-1400"> int numFailedCase = 0;</span>
<span class="source-line-no">1401</span><span id="line-1401"> FSTableDescriptors fstd = new FSTableDescriptors(getConf());</span>
<span class="source-line-no">1402</span><span id="line-1402"> while (iter.hasNext()) {</span>
<span class="source-line-no">1403</span><span id="line-1403"> Entry&lt;TableName, Set&lt;String&gt;&gt; entry = iter.next();</span>
<span class="source-line-no">1404</span><span id="line-1404"> TableName tableName = entry.getKey();</span>
<span class="source-line-no">1405</span><span id="line-1405"> LOG.info("Trying to fix orphan table error: " + tableName);</span>
<span class="source-line-no">1406</span><span id="line-1406"> if (j &lt; htds.length) {</span>
<span class="source-line-no">1407</span><span id="line-1407"> if (tableName.equals(htds[j].getTableName())) {</span>
<span class="source-line-no">1408</span><span id="line-1408"> TableDescriptor htd = htds[j];</span>
<span class="source-line-no">1409</span><span id="line-1409"> LOG.info("fixing orphan table: " + tableName + " from cache");</span>
<span class="source-line-no">1410</span><span id="line-1410"> fstd.createTableDescriptor(htd, true);</span>
<span class="source-line-no">1411</span><span id="line-1411"> j++;</span>
<span class="source-line-no">1412</span><span id="line-1412"> iter.remove();</span>
<span class="source-line-no">1413</span><span id="line-1413"> }</span>
<span class="source-line-no">1414</span><span id="line-1414"> } else {</span>
<span class="source-line-no">1415</span><span id="line-1415"> if (fabricateTableInfo(fstd, tableName, entry.getValue())) {</span>
<span class="source-line-no">1416</span><span id="line-1416"> LOG.warn("fixing orphan table: " + tableName + " with a default .tableinfo file");</span>
<span class="source-line-no">1417</span><span id="line-1417"> LOG.warn(</span>
<span class="source-line-no">1418</span><span id="line-1418"> "Strongly recommend to modify the TableDescriptor if necessary for: " + tableName);</span>
<span class="source-line-no">1419</span><span id="line-1419"> iter.remove();</span>
<span class="source-line-no">1420</span><span id="line-1420"> } else {</span>
<span class="source-line-no">1421</span><span id="line-1421"> LOG.error("Unable to create default .tableinfo for " + tableName</span>
<span class="source-line-no">1422</span><span id="line-1422"> + " while missing column family information");</span>
<span class="source-line-no">1423</span><span id="line-1423"> numFailedCase++;</span>
<span class="source-line-no">1424</span><span id="line-1424"> }</span>
<span class="source-line-no">1425</span><span id="line-1425"> }</span>
<span class="source-line-no">1426</span><span id="line-1426"> fixes++;</span>
<span class="source-line-no">1427</span><span id="line-1427"> }</span>
<span class="source-line-no">1428</span><span id="line-1428"></span>
<span class="source-line-no">1429</span><span id="line-1429"> if (orphanTableDirs.isEmpty()) {</span>
<span class="source-line-no">1430</span><span id="line-1430"> // all orphanTableDirs are luckily recovered</span>
<span class="source-line-no">1431</span><span id="line-1431"> // re-run doFsck after recovering the .tableinfo file</span>
<span class="source-line-no">1432</span><span id="line-1432"> setShouldRerun();</span>
<span class="source-line-no">1433</span><span id="line-1433"> LOG.warn(</span>
<span class="source-line-no">1434</span><span id="line-1434"> "Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed");</span>
<span class="source-line-no">1435</span><span id="line-1435"> } else if (numFailedCase &gt; 0) {</span>
<span class="source-line-no">1436</span><span id="line-1436"> LOG.error("Failed to fix " + numFailedCase + " OrphanTables with default .tableinfo files");</span>
<span class="source-line-no">1437</span><span id="line-1437"> }</span>
<span class="source-line-no">1438</span><span id="line-1438"></span>
<span class="source-line-no">1439</span><span id="line-1439"> }</span>
<span class="source-line-no">1440</span><span id="line-1440"> // cleanup the list</span>
<span class="source-line-no">1441</span><span id="line-1441"> orphanTableDirs.clear();</span>
<span class="source-line-no">1442</span><span id="line-1442"></span>
<span class="source-line-no">1443</span><span id="line-1443"> }</span>
<span class="source-line-no">1444</span><span id="line-1444"></span>
<span class="source-line-no">1445</span><span id="line-1445"> /**</span>
<span class="source-line-no">1446</span><span id="line-1446"> * Log an appropriate message about whether or not overlapping merges are computed in parallel.</span>
<span class="source-line-no">1447</span><span id="line-1447"> */</span>
<span class="source-line-no">1448</span><span id="line-1448"> private void logParallelMerge() {</span>
<span class="source-line-no">1449</span><span id="line-1449"> if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) {</span>
<span class="source-line-no">1450</span><span id="line-1450"> LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to"</span>
<span class="source-line-no">1451</span><span id="line-1451"> + " false to run serially.");</span>
<span class="source-line-no">1452</span><span id="line-1452"> } else {</span>
<span class="source-line-no">1453</span><span id="line-1453"> LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to"</span>
<span class="source-line-no">1454</span><span id="line-1454"> + " true to run in parallel.");</span>
<span class="source-line-no">1455</span><span id="line-1455"> }</span>
<span class="source-line-no">1456</span><span id="line-1456"> }</span>
<span class="source-line-no">1457</span><span id="line-1457"></span>
<span class="source-line-no">1458</span><span id="line-1458"> private SortedMap&lt;TableName, HbckTableInfo&gt; checkHdfsIntegrity(boolean fixHoles,</span>
<span class="source-line-no">1459</span><span id="line-1459"> boolean fixOverlaps) throws IOException {</span>
<span class="source-line-no">1460</span><span id="line-1460"> LOG.info("Checking HBase region split map from HDFS data...");</span>
<span class="source-line-no">1461</span><span id="line-1461"> logParallelMerge();</span>
<span class="source-line-no">1462</span><span id="line-1462"> for (HbckTableInfo tInfo : tablesInfo.values()) {</span>
<span class="source-line-no">1463</span><span id="line-1463"> TableIntegrityErrorHandler handler;</span>
<span class="source-line-no">1464</span><span id="line-1464"> if (fixHoles || fixOverlaps) {</span>
<span class="source-line-no">1465</span><span id="line-1465"> handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(), fixHoles, fixOverlaps);</span>
<span class="source-line-no">1466</span><span id="line-1466"> } else {</span>
<span class="source-line-no">1467</span><span id="line-1467"> handler = tInfo.new IntegrityFixSuggester(tInfo, errors);</span>
<span class="source-line-no">1468</span><span id="line-1468"> }</span>
<span class="source-line-no">1469</span><span id="line-1469"> if (!tInfo.checkRegionChain(handler)) {</span>
<span class="source-line-no">1470</span><span id="line-1470"> // should dump info as well.</span>
<span class="source-line-no">1471</span><span id="line-1471"> errors.report("Found inconsistency in table " + tInfo.getName());</span>
<span class="source-line-no">1472</span><span id="line-1472"> }</span>
<span class="source-line-no">1473</span><span id="line-1473"> }</span>
<span class="source-line-no">1474</span><span id="line-1474"> return tablesInfo;</span>
<span class="source-line-no">1475</span><span id="line-1475"> }</span>
<span class="source-line-no">1476</span><span id="line-1476"></span>
<span class="source-line-no">1477</span><span id="line-1477"> Path getSidelineDir() throws IOException {</span>
<span class="source-line-no">1478</span><span id="line-1478"> if (sidelineDir == null) {</span>
<span class="source-line-no">1479</span><span id="line-1479"> Path hbaseDir = CommonFSUtils.getRootDir(getConf());</span>
<span class="source-line-no">1480</span><span id="line-1480"> Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);</span>
<span class="source-line-no">1481</span><span id="line-1481"> sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-" + startMillis);</span>
<span class="source-line-no">1482</span><span id="line-1482"> }</span>
<span class="source-line-no">1483</span><span id="line-1483"> return sidelineDir;</span>
<span class="source-line-no">1484</span><span id="line-1484"> }</span>
<span class="source-line-no">1485</span><span id="line-1485"></span>
<span class="source-line-no">1486</span><span id="line-1486"> /**</span>
<span class="source-line-no">1487</span><span id="line-1487"> * Sideline a region dir (instead of deleting it)</span>
<span class="source-line-no">1488</span><span id="line-1488"> */</span>
<span class="source-line-no">1489</span><span id="line-1489"> Path sidelineRegionDir(FileSystem fs, HbckRegionInfo hi) throws IOException {</span>
<span class="source-line-no">1490</span><span id="line-1490"> return sidelineRegionDir(fs, null, hi);</span>
<span class="source-line-no">1491</span><span id="line-1491"> }</span>
<span class="source-line-no">1492</span><span id="line-1492"></span>
<span class="source-line-no">1493</span><span id="line-1493"> /**</span>
<span class="source-line-no">1494</span><span id="line-1494"> * Sideline a region dir (instead of deleting it)</span>
<span class="source-line-no">1495</span><span id="line-1495"> * @param parentDir if specified, the region will be sidelined to folder like</span>
<span class="source-line-no">1496</span><span id="line-1496"> * {@literal .../parentDir/&lt;table name&gt;/&lt;region name&gt;}. The purpose is to group</span>
<span class="source-line-no">1497</span><span id="line-1497"> * together similar regions sidelined, for example, those regions should be bulk</span>
<span class="source-line-no">1498</span><span id="line-1498"> * loaded back later on. If NULL, it is ignored.</span>
<span class="source-line-no">1499</span><span id="line-1499"> */</span>
<span class="source-line-no">1500</span><span id="line-1500"> Path sidelineRegionDir(FileSystem fs, String parentDir, HbckRegionInfo hi) throws IOException {</span>
<span class="source-line-no">1501</span><span id="line-1501"> TableName tableName = hi.getTableName();</span>
<span class="source-line-no">1502</span><span id="line-1502"> Path regionDir = hi.getHdfsRegionDir();</span>
<span class="source-line-no">1503</span><span id="line-1503"></span>
<span class="source-line-no">1504</span><span id="line-1504"> if (!fs.exists(regionDir)) {</span>
<span class="source-line-no">1505</span><span id="line-1505"> LOG.warn("No previous " + regionDir + " exists. Continuing.");</span>
<span class="source-line-no">1506</span><span id="line-1506"> return null;</span>
<span class="source-line-no">1507</span><span id="line-1507"> }</span>
<span class="source-line-no">1508</span><span id="line-1508"></span>
<span class="source-line-no">1509</span><span id="line-1509"> Path rootDir = getSidelineDir();</span>
<span class="source-line-no">1510</span><span id="line-1510"> if (parentDir != null) {</span>
<span class="source-line-no">1511</span><span id="line-1511"> rootDir = new Path(rootDir, parentDir);</span>
<span class="source-line-no">1512</span><span id="line-1512"> }</span>
<span class="source-line-no">1513</span><span id="line-1513"> Path sidelineTableDir = CommonFSUtils.getTableDir(rootDir, tableName);</span>
<span class="source-line-no">1514</span><span id="line-1514"> Path sidelineRegionDir = new Path(sidelineTableDir, regionDir.getName());</span>
<span class="source-line-no">1515</span><span id="line-1515"> fs.mkdirs(sidelineRegionDir);</span>
<span class="source-line-no">1516</span><span id="line-1516"> boolean success = false;</span>
<span class="source-line-no">1517</span><span id="line-1517"> FileStatus[] cfs = fs.listStatus(regionDir);</span>
<span class="source-line-no">1518</span><span id="line-1518"> if (cfs == null) {</span>
<span class="source-line-no">1519</span><span id="line-1519"> LOG.info("Region dir is empty: " + regionDir);</span>
<span class="source-line-no">1520</span><span id="line-1520"> } else {</span>
<span class="source-line-no">1521</span><span id="line-1521"> for (FileStatus cf : cfs) {</span>
<span class="source-line-no">1522</span><span id="line-1522"> Path src = cf.getPath();</span>
<span class="source-line-no">1523</span><span id="line-1523"> Path dst = new Path(sidelineRegionDir, src.getName());</span>
<span class="source-line-no">1524</span><span id="line-1524"> if (fs.isFile(src)) {</span>
<span class="source-line-no">1525</span><span id="line-1525"> // simple file</span>
<span class="source-line-no">1526</span><span id="line-1526"> success = fs.rename(src, dst);</span>
<span class="source-line-no">1527</span><span id="line-1527"> if (!success) {</span>
<span class="source-line-no">1528</span><span id="line-1528"> String msg = "Unable to rename file " + src + " to " + dst;</span>
<span class="source-line-no">1529</span><span id="line-1529"> LOG.error(msg);</span>
<span class="source-line-no">1530</span><span id="line-1530"> throw new IOException(msg);</span>
<span class="source-line-no">1531</span><span id="line-1531"> }</span>
<span class="source-line-no">1532</span><span id="line-1532"> continue;</span>
<span class="source-line-no">1533</span><span id="line-1533"> }</span>
<span class="source-line-no">1534</span><span id="line-1534"></span>
<span class="source-line-no">1535</span><span id="line-1535"> // is a directory.</span>
<span class="source-line-no">1536</span><span id="line-1536"> fs.mkdirs(dst);</span>
<span class="source-line-no">1537</span><span id="line-1537"></span>
<span class="source-line-no">1538</span><span id="line-1538"> LOG.info("Sidelining files from " + src + " into containing region " + dst);</span>
<span class="source-line-no">1539</span><span id="line-1539"> // FileSystem.rename is inconsistent with directories -- if the</span>
<span class="source-line-no">1540</span><span id="line-1540"> // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,</span>
<span class="source-line-no">1541</span><span id="line-1541"> // it moves the src into the dst dir resulting in (foo/a/b). If</span>
<span class="source-line-no">1542</span><span id="line-1542"> // the dst does not exist, and the src a dir, src becomes dst. (foo/b)</span>
<span class="source-line-no">1543</span><span id="line-1543"> FileStatus[] hfiles = fs.listStatus(src);</span>
<span class="source-line-no">1544</span><span id="line-1544"> if (hfiles != null &amp;&amp; hfiles.length &gt; 0) {</span>
<span class="source-line-no">1545</span><span id="line-1545"> for (FileStatus hfile : hfiles) {</span>
<span class="source-line-no">1546</span><span id="line-1546"> success = fs.rename(hfile.getPath(), dst);</span>
<span class="source-line-no">1547</span><span id="line-1547"> if (!success) {</span>
<span class="source-line-no">1548</span><span id="line-1548"> String msg = "Unable to rename file " + src + " to " + dst;</span>
<span class="source-line-no">1549</span><span id="line-1549"> LOG.error(msg);</span>
<span class="source-line-no">1550</span><span id="line-1550"> throw new IOException(msg);</span>
<span class="source-line-no">1551</span><span id="line-1551"> }</span>
<span class="source-line-no">1552</span><span id="line-1552"> }</span>
<span class="source-line-no">1553</span><span id="line-1553"> }</span>
<span class="source-line-no">1554</span><span id="line-1554"> LOG.debug("Sideline directory contents:");</span>
<span class="source-line-no">1555</span><span id="line-1555"> debugLsr(sidelineRegionDir);</span>
<span class="source-line-no">1556</span><span id="line-1556"> }</span>
<span class="source-line-no">1557</span><span id="line-1557"> }</span>
<span class="source-line-no">1558</span><span id="line-1558"></span>
<span class="source-line-no">1559</span><span id="line-1559"> LOG.info("Removing old region dir: " + regionDir);</span>
<span class="source-line-no">1560</span><span id="line-1560"> success = fs.delete(regionDir, true);</span>
<span class="source-line-no">1561</span><span id="line-1561"> if (!success) {</span>
<span class="source-line-no">1562</span><span id="line-1562"> String msg = "Unable to delete dir " + regionDir;</span>
<span class="source-line-no">1563</span><span id="line-1563"> LOG.error(msg);</span>
<span class="source-line-no">1564</span><span id="line-1564"> throw new IOException(msg);</span>
<span class="source-line-no">1565</span><span id="line-1565"> }</span>
<span class="source-line-no">1566</span><span id="line-1566"> return sidelineRegionDir;</span>
<span class="source-line-no">1567</span><span id="line-1567"> }</span>
<span class="source-line-no">1568</span><span id="line-1568"></span>
<span class="source-line-no">1569</span><span id="line-1569"> /**</span>
<span class="source-line-no">1570</span><span id="line-1570"> * Load the list of disabled tables in ZK into local set.</span>
<span class="source-line-no">1571</span><span id="line-1571"> */</span>
<span class="source-line-no">1572</span><span id="line-1572"> private void loadTableStates() throws IOException {</span>
<span class="source-line-no">1573</span><span id="line-1573"> tableStates = MetaTableAccessor.getTableStates(connection);</span>
<span class="source-line-no">1574</span><span id="line-1574"> // Add hbase:meta so this tool keeps working. In hbase2, meta is always enabled though it</span>
<span class="source-line-no">1575</span><span id="line-1575"> // has no entry in the table states. HBCK doesn't work right w/ hbase2 but just do this in</span>
<span class="source-line-no">1576</span><span id="line-1576"> // meantime.</span>
<span class="source-line-no">1577</span><span id="line-1577"> this.tableStates.put(TableName.META_TABLE_NAME,</span>
<span class="source-line-no">1578</span><span id="line-1578"> new TableState(TableName.META_TABLE_NAME, TableState.State.ENABLED));</span>
<span class="source-line-no">1579</span><span id="line-1579"> }</span>
<span class="source-line-no">1580</span><span id="line-1580"></span>
<span class="source-line-no">1581</span><span id="line-1581"> /**</span>
<span class="source-line-no">1582</span><span id="line-1582"> * Check if the specified region's table is disabled.</span>
<span class="source-line-no">1583</span><span id="line-1583"> * @param tableName table to check status of</span>
<span class="source-line-no">1584</span><span id="line-1584"> */</span>
<span class="source-line-no">1585</span><span id="line-1585"> boolean isTableDisabled(TableName tableName) {</span>
<span class="source-line-no">1586</span><span id="line-1586"> return tableStates.containsKey(tableName)</span>
<span class="source-line-no">1587</span><span id="line-1587"> &amp;&amp; tableStates.get(tableName).inStates(TableState.State.DISABLED, TableState.State.DISABLING);</span>
<span class="source-line-no">1588</span><span id="line-1588"> }</span>
<span class="source-line-no">1589</span><span id="line-1589"></span>
<span class="source-line-no">1590</span><span id="line-1590"> /**</span>
<span class="source-line-no">1591</span><span id="line-1591"> * Scan HDFS for all regions, recording their information into regionInfoMap</span>
<span class="source-line-no">1592</span><span id="line-1592"> */</span>
<span class="source-line-no">1593</span><span id="line-1593"> public void loadHdfsRegionDirs() throws IOException, InterruptedException {</span>
<span class="source-line-no">1594</span><span id="line-1594"> Path rootDir = CommonFSUtils.getRootDir(getConf());</span>
<span class="source-line-no">1595</span><span id="line-1595"> FileSystem fs = rootDir.getFileSystem(getConf());</span>
<span class="source-line-no">1596</span><span id="line-1596"></span>
<span class="source-line-no">1597</span><span id="line-1597"> // list all tables from HDFS</span>
<span class="source-line-no">1598</span><span id="line-1598"> List&lt;FileStatus&gt; tableDirs = Lists.newArrayList();</span>
<span class="source-line-no">1599</span><span id="line-1599"></span>
<span class="source-line-no">1600</span><span id="line-1600"> boolean foundVersionFile = fs.exists(new Path(rootDir, HConstants.VERSION_FILE_NAME));</span>
<span class="source-line-no">1601</span><span id="line-1601"></span>
<span class="source-line-no">1602</span><span id="line-1602"> List&lt;Path&gt; paths = FSUtils.getTableDirs(fs, rootDir);</span>
<span class="source-line-no">1603</span><span id="line-1603"> for (Path path : paths) {</span>
<span class="source-line-no">1604</span><span id="line-1604"> TableName tableName = CommonFSUtils.getTableName(path);</span>
<span class="source-line-no">1605</span><span id="line-1605"> if (</span>
<span class="source-line-no">1606</span><span id="line-1606"> (!checkMetaOnly &amp;&amp; isTableIncluded(tableName))</span>
<span class="source-line-no">1607</span><span id="line-1607"> || tableName.equals(TableName.META_TABLE_NAME)</span>
<span class="source-line-no">1608</span><span id="line-1608"> ) {</span>
<span class="source-line-no">1609</span><span id="line-1609"> tableDirs.add(fs.getFileStatus(path));</span>
<span class="source-line-no">1610</span><span id="line-1610"> }</span>
<span class="source-line-no">1611</span><span id="line-1611"> }</span>
<span class="source-line-no">1612</span><span id="line-1612"></span>
<span class="source-line-no">1613</span><span id="line-1613"> // verify that version file exists</span>
<span class="source-line-no">1614</span><span id="line-1614"> if (!foundVersionFile) {</span>
<span class="source-line-no">1615</span><span id="line-1615"> errors.reportError(ERROR_CODE.NO_VERSION_FILE,</span>
<span class="source-line-no">1616</span><span id="line-1616"> "Version file does not exist in root dir " + rootDir);</span>
<span class="source-line-no">1617</span><span id="line-1617"> if (shouldFixVersionFile()) {</span>
<span class="source-line-no">1618</span><span id="line-1618"> LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME + " file.");</span>
<span class="source-line-no">1619</span><span id="line-1619"> setShouldRerun();</span>
<span class="source-line-no">1620</span><span id="line-1620"> FSUtils.setVersion(fs, rootDir,</span>
<span class="source-line-no">1621</span><span id="line-1621"> getConf().getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000),</span>
<span class="source-line-no">1622</span><span id="line-1622"> getConf().getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,</span>
<span class="source-line-no">1623</span><span id="line-1623"> HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));</span>
<span class="source-line-no">1624</span><span id="line-1624"> }</span>
<span class="source-line-no">1625</span><span id="line-1625"> }</span>
<span class="source-line-no">1626</span><span id="line-1626"></span>
<span class="source-line-no">1627</span><span id="line-1627"> // Avoid multithreading at table-level because already multithreaded internally at</span>
<span class="source-line-no">1628</span><span id="line-1628"> // region-level. Additionally multithreading at table-level can lead to deadlock</span>
<span class="source-line-no">1629</span><span id="line-1629"> // if there are many tables in the cluster. Since there are a limited # of threads</span>
<span class="source-line-no">1630</span><span id="line-1630"> // in the executor's thread pool and if we multithread at the table-level by putting</span>
<span class="source-line-no">1631</span><span id="line-1631"> // WorkItemHdfsDir callables into the executor, then we will have some threads in the</span>
<span class="source-line-no">1632</span><span id="line-1632"> // executor tied up solely in waiting for the tables' region-level calls to complete.</span>
<span class="source-line-no">1633</span><span id="line-1633"> // If there are enough tables then there will be no actual threads in the pool left</span>
<span class="source-line-no">1634</span><span id="line-1634"> // for the region-level callables to be serviced.</span>
<span class="source-line-no">1635</span><span id="line-1635"> for (FileStatus tableDir : tableDirs) {</span>
<span class="source-line-no">1636</span><span id="line-1636"> LOG.debug("Loading region dirs from " + tableDir.getPath());</span>
<span class="source-line-no">1637</span><span id="line-1637"> WorkItemHdfsDir item = new WorkItemHdfsDir(fs, errors, tableDir);</span>
<span class="source-line-no">1638</span><span id="line-1638"> try {</span>
<span class="source-line-no">1639</span><span id="line-1639"> item.call();</span>
<span class="source-line-no">1640</span><span id="line-1640"> } catch (ExecutionException e) {</span>
<span class="source-line-no">1641</span><span id="line-1641"> LOG.warn("Could not completely load table dir " + tableDir.getPath(), e.getCause());</span>
<span class="source-line-no">1642</span><span id="line-1642"> }</span>
<span class="source-line-no">1643</span><span id="line-1643"> }</span>
<span class="source-line-no">1644</span><span id="line-1644"> errors.print("");</span>
<span class="source-line-no">1645</span><span id="line-1645"> }</span>
<span class="source-line-no">1646</span><span id="line-1646"></span>
<span class="source-line-no">1647</span><span id="line-1647"> /**</span>
<span class="source-line-no">1648</span><span id="line-1648"> * Record the location of the hbase:meta region as found in ZooKeeper.</span>
<span class="source-line-no">1649</span><span id="line-1649"> */</span>
<span class="source-line-no">1650</span><span id="line-1650"> private boolean recordMetaRegion() throws IOException {</span>
<span class="source-line-no">1651</span><span id="line-1651"> List&lt;HRegionLocation&gt; locs;</span>
<span class="source-line-no">1652</span><span id="line-1652"> try (RegionLocator locator = connection.getRegionLocator(TableName.META_TABLE_NAME)) {</span>
<span class="source-line-no">1653</span><span id="line-1653"> locs = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true);</span>
<span class="source-line-no">1654</span><span id="line-1654"> }</span>
<span class="source-line-no">1655</span><span id="line-1655"> if (locs == null || locs.isEmpty()) {</span>
<span class="source-line-no">1656</span><span id="line-1656"> errors.reportError(ERROR_CODE.NULL_META_REGION, "META region was not found in ZooKeeper");</span>
<span class="source-line-no">1657</span><span id="line-1657"> return false;</span>
<span class="source-line-no">1658</span><span id="line-1658"> }</span>
<span class="source-line-no">1659</span><span id="line-1659"> for (HRegionLocation metaLocation : locs) {</span>
<span class="source-line-no">1660</span><span id="line-1660"> // Check if Meta region is valid and existing</span>
<span class="source-line-no">1661</span><span id="line-1661"> if (metaLocation == null) {</span>
<span class="source-line-no">1662</span><span id="line-1662"> errors.reportError(ERROR_CODE.NULL_META_REGION, "META region location is null");</span>
<span class="source-line-no">1663</span><span id="line-1663"> return false;</span>
<span class="source-line-no">1664</span><span id="line-1664"> }</span>
<span class="source-line-no">1665</span><span id="line-1665"> if (metaLocation.getRegion() == null) {</span>
<span class="source-line-no">1666</span><span id="line-1666"> errors.reportError(ERROR_CODE.NULL_META_REGION, "META location regionInfo is null");</span>
<span class="source-line-no">1667</span><span id="line-1667"> return false;</span>
<span class="source-line-no">1668</span><span id="line-1668"> }</span>
<span class="source-line-no">1669</span><span id="line-1669"> if (metaLocation.getHostname() == null) {</span>
<span class="source-line-no">1670</span><span id="line-1670"> errors.reportError(ERROR_CODE.NULL_META_REGION, "META location hostName is null");</span>
<span class="source-line-no">1671</span><span id="line-1671"> return false;</span>
<span class="source-line-no">1672</span><span id="line-1672"> }</span>
<span class="source-line-no">1673</span><span id="line-1673"> ServerName sn = metaLocation.getServerName();</span>
<span class="source-line-no">1674</span><span id="line-1674"> HbckRegionInfo.MetaEntry m = new HbckRegionInfo.MetaEntry(metaLocation.getRegion(), sn,</span>
<span class="source-line-no">1675</span><span id="line-1675"> EnvironmentEdgeManager.currentTime());</span>
<span class="source-line-no">1676</span><span id="line-1676"> HbckRegionInfo hbckRegionInfo = regionInfoMap.get(metaLocation.getRegion().getEncodedName());</span>
<span class="source-line-no">1677</span><span id="line-1677"> if (hbckRegionInfo == null) {</span>
<span class="source-line-no">1678</span><span id="line-1678"> regionInfoMap.put(metaLocation.getRegion().getEncodedName(), new HbckRegionInfo(m));</span>
<span class="source-line-no">1679</span><span id="line-1679"> } else {</span>
<span class="source-line-no">1680</span><span id="line-1680"> hbckRegionInfo.setMetaEntry(m);</span>
<span class="source-line-no">1681</span><span id="line-1681"> }</span>
<span class="source-line-no">1682</span><span id="line-1682"> }</span>
<span class="source-line-no">1683</span><span id="line-1683"> return true;</span>
<span class="source-line-no">1684</span><span id="line-1684"> }</span>
<span class="source-line-no">1685</span><span id="line-1685"></span>
<span class="source-line-no">1686</span><span id="line-1686"> private ZKWatcher createZooKeeperWatcher() throws IOException {</span>
<span class="source-line-no">1687</span><span id="line-1687"> return new ZKWatcher(getConf(), "hbase Fsck", new Abortable() {</span>
<span class="source-line-no">1688</span><span id="line-1688"> @Override</span>
<span class="source-line-no">1689</span><span id="line-1689"> public void abort(String why, Throwable e) {</span>
<span class="source-line-no">1690</span><span id="line-1690"> LOG.error(why, e);</span>
<span class="source-line-no">1691</span><span id="line-1691"> System.exit(1);</span>
<span class="source-line-no">1692</span><span id="line-1692"> }</span>
<span class="source-line-no">1693</span><span id="line-1693"></span>
<span class="source-line-no">1694</span><span id="line-1694"> @Override</span>
<span class="source-line-no">1695</span><span id="line-1695"> public boolean isAborted() {</span>
<span class="source-line-no">1696</span><span id="line-1696"> return false;</span>
<span class="source-line-no">1697</span><span id="line-1697"> }</span>
<span class="source-line-no">1698</span><span id="line-1698"></span>
<span class="source-line-no">1699</span><span id="line-1699"> });</span>
<span class="source-line-no">1700</span><span id="line-1700"> }</span>
<span class="source-line-no">1701</span><span id="line-1701"></span>
<span class="source-line-no">1702</span><span id="line-1702"> /**</span>
<span class="source-line-no">1703</span><span id="line-1703"> * Contacts each regionserver and fetches metadata about regions.</span>
<span class="source-line-no">1704</span><span id="line-1704"> * @param regionServerList - the list of region servers to connect to</span>
<span class="source-line-no">1705</span><span id="line-1705"> * @throws IOException if a remote or network exception occurs</span>
<span class="source-line-no">1706</span><span id="line-1706"> */</span>
<span class="source-line-no">1707</span><span id="line-1707"> void processRegionServers(Collection&lt;ServerName&gt; regionServerList)</span>
<span class="source-line-no">1708</span><span id="line-1708"> throws IOException, InterruptedException {</span>
<span class="source-line-no">1709</span><span id="line-1709"></span>
<span class="source-line-no">1710</span><span id="line-1710"> List&lt;WorkItemRegion&gt; workItems = new ArrayList&lt;&gt;(regionServerList.size());</span>
<span class="source-line-no">1711</span><span id="line-1711"> List&lt;Future&lt;Void&gt;&gt; workFutures;</span>
<span class="source-line-no">1712</span><span id="line-1712"></span>
<span class="source-line-no">1713</span><span id="line-1713"> // loop to contact each region server in parallel</span>
<span class="source-line-no">1714</span><span id="line-1714"> for (ServerName rsinfo : regionServerList) {</span>
<span class="source-line-no">1715</span><span id="line-1715"> workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));</span>
<span class="source-line-no">1716</span><span id="line-1716"> }</span>
<span class="source-line-no">1717</span><span id="line-1717"></span>
<span class="source-line-no">1718</span><span id="line-1718"> workFutures = executor.invokeAll(workItems);</span>
<span class="source-line-no">1719</span><span id="line-1719"></span>
<span class="source-line-no">1720</span><span id="line-1720"> for (int i = 0; i &lt; workFutures.size(); i++) {</span>
<span class="source-line-no">1721</span><span id="line-1721"> WorkItemRegion item = workItems.get(i);</span>
<span class="source-line-no">1722</span><span id="line-1722"> Future&lt;Void&gt; f = workFutures.get(i);</span>
<span class="source-line-no">1723</span><span id="line-1723"> try {</span>
<span class="source-line-no">1724</span><span id="line-1724"> f.get();</span>
<span class="source-line-no">1725</span><span id="line-1725"> } catch (ExecutionException e) {</span>
<span class="source-line-no">1726</span><span id="line-1726"> LOG.warn("Could not process regionserver {}", item.rsinfo.getAddress(), e.getCause());</span>
<span class="source-line-no">1727</span><span id="line-1727"> }</span>
<span class="source-line-no">1728</span><span id="line-1728"> }</span>
<span class="source-line-no">1729</span><span id="line-1729"> }</span>
<span class="source-line-no">1730</span><span id="line-1730"></span>
<span class="source-line-no">1731</span><span id="line-1731"> /**</span>
<span class="source-line-no">1732</span><span id="line-1732"> * Check consistency of all regions that have been found in previous phases.</span>
<span class="source-line-no">1733</span><span id="line-1733"> */</span>
<span class="source-line-no">1734</span><span id="line-1734"> private void checkAndFixConsistency() throws IOException, KeeperException, InterruptedException {</span>
<span class="source-line-no">1735</span><span id="line-1735"> // Divide the checks in two phases. One for default/primary replicas and another</span>
<span class="source-line-no">1736</span><span id="line-1736"> // for the non-primary ones. Keeps code cleaner this way.</span>
<span class="source-line-no">1737</span><span id="line-1737"></span>
<span class="source-line-no">1738</span><span id="line-1738"> List&lt;CheckRegionConsistencyWorkItem&gt; workItems = new ArrayList&lt;&gt;(regionInfoMap.size());</span>
<span class="source-line-no">1739</span><span id="line-1739"> for (java.util.Map.Entry&lt;String, HbckRegionInfo&gt; e : regionInfoMap.entrySet()) {</span>
<span class="source-line-no">1740</span><span id="line-1740"> if (e.getValue().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">1741</span><span id="line-1741"> workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));</span>
<span class="source-line-no">1742</span><span id="line-1742"> }</span>
<span class="source-line-no">1743</span><span id="line-1743"> }</span>
<span class="source-line-no">1744</span><span id="line-1744"> checkRegionConsistencyConcurrently(workItems);</span>
<span class="source-line-no">1745</span><span id="line-1745"></span>
<span class="source-line-no">1746</span><span id="line-1746"> boolean prevHdfsCheck = shouldCheckHdfs();</span>
<span class="source-line-no">1747</span><span id="line-1747"> setCheckHdfs(false); // replicas don't have any hdfs data</span>
<span class="source-line-no">1748</span><span id="line-1748"> // Run a pass over the replicas and fix any assignment issues that exist on the currently</span>
<span class="source-line-no">1749</span><span id="line-1749"> // deployed/undeployed replicas.</span>
<span class="source-line-no">1750</span><span id="line-1750"> List&lt;CheckRegionConsistencyWorkItem&gt; replicaWorkItems = new ArrayList&lt;&gt;(regionInfoMap.size());</span>
<span class="source-line-no">1751</span><span id="line-1751"> for (java.util.Map.Entry&lt;String, HbckRegionInfo&gt; e : regionInfoMap.entrySet()) {</span>
<span class="source-line-no">1752</span><span id="line-1752"> if (e.getValue().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">1753</span><span id="line-1753"> replicaWorkItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue()));</span>
<span class="source-line-no">1754</span><span id="line-1754"> }</span>
<span class="source-line-no">1755</span><span id="line-1755"> }</span>
<span class="source-line-no">1756</span><span id="line-1756"> checkRegionConsistencyConcurrently(replicaWorkItems);</span>
<span class="source-line-no">1757</span><span id="line-1757"> setCheckHdfs(prevHdfsCheck);</span>
<span class="source-line-no">1758</span><span id="line-1758"></span>
<span class="source-line-no">1759</span><span id="line-1759"> // If some regions is skipped during checkRegionConsistencyConcurrently() phase, we might</span>
<span class="source-line-no">1760</span><span id="line-1760"> // not get accurate state of the hbase if continuing. The config here allows users to tune</span>
<span class="source-line-no">1761</span><span id="line-1761"> // the tolerance of number of skipped region.</span>
<span class="source-line-no">1762</span><span id="line-1762"> // TODO: evaluate the consequence to continue the hbck operation without config.</span>
<span class="source-line-no">1763</span><span id="line-1763"> int terminateThreshold = getConf().getInt("hbase.hbck.skipped.regions.limit", 0);</span>
<span class="source-line-no">1764</span><span id="line-1764"> int numOfSkippedRegions = skippedRegions.size();</span>
<span class="source-line-no">1765</span><span id="line-1765"> if (numOfSkippedRegions &gt; 0 &amp;&amp; numOfSkippedRegions &gt; terminateThreshold) {</span>
<span class="source-line-no">1766</span><span id="line-1766"> throw new IOException(</span>
<span class="source-line-no">1767</span><span id="line-1767"> numOfSkippedRegions + " region(s) could not be checked or repaired. See logs for detail.");</span>
<span class="source-line-no">1768</span><span id="line-1768"> }</span>
<span class="source-line-no">1769</span><span id="line-1769"></span>
<span class="source-line-no">1770</span><span id="line-1770"> if (shouldCheckHdfs()) {</span>
<span class="source-line-no">1771</span><span id="line-1771"> checkAndFixTableStates();</span>
<span class="source-line-no">1772</span><span id="line-1772"> }</span>
<span class="source-line-no">1773</span><span id="line-1773"> }</span>
<span class="source-line-no">1774</span><span id="line-1774"></span>
<span class="source-line-no">1775</span><span id="line-1775"> /**</span>
<span class="source-line-no">1776</span><span id="line-1776"> * Check consistency of all regions using multiple threads concurrently.</span>
<span class="source-line-no">1777</span><span id="line-1777"> */</span>
<span class="source-line-no">1778</span><span id="line-1778"> private void</span>
<span class="source-line-no">1779</span><span id="line-1779"> checkRegionConsistencyConcurrently(final List&lt;CheckRegionConsistencyWorkItem&gt; workItems)</span>
<span class="source-line-no">1780</span><span id="line-1780"> throws IOException, KeeperException, InterruptedException {</span>
<span class="source-line-no">1781</span><span id="line-1781"> if (workItems.isEmpty()) {</span>
<span class="source-line-no">1782</span><span id="line-1782"> return; // nothing to check</span>
<span class="source-line-no">1783</span><span id="line-1783"> }</span>
<span class="source-line-no">1784</span><span id="line-1784"></span>
<span class="source-line-no">1785</span><span id="line-1785"> List&lt;Future&lt;Void&gt;&gt; workFutures = executor.invokeAll(workItems);</span>
<span class="source-line-no">1786</span><span id="line-1786"> for (Future&lt;Void&gt; f : workFutures) {</span>
<span class="source-line-no">1787</span><span id="line-1787"> try {</span>
<span class="source-line-no">1788</span><span id="line-1788"> f.get();</span>
<span class="source-line-no">1789</span><span id="line-1789"> } catch (ExecutionException e1) {</span>
<span class="source-line-no">1790</span><span id="line-1790"> LOG.warn("Could not check region consistency ", e1.getCause());</span>
<span class="source-line-no">1791</span><span id="line-1791"> if (e1.getCause() instanceof IOException) {</span>
<span class="source-line-no">1792</span><span id="line-1792"> throw (IOException) e1.getCause();</span>
<span class="source-line-no">1793</span><span id="line-1793"> } else if (e1.getCause() instanceof KeeperException) {</span>
<span class="source-line-no">1794</span><span id="line-1794"> throw (KeeperException) e1.getCause();</span>
<span class="source-line-no">1795</span><span id="line-1795"> } else if (e1.getCause() instanceof InterruptedException) {</span>
<span class="source-line-no">1796</span><span id="line-1796"> throw (InterruptedException) e1.getCause();</span>
<span class="source-line-no">1797</span><span id="line-1797"> } else {</span>
<span class="source-line-no">1798</span><span id="line-1798"> throw new IOException(e1.getCause());</span>
<span class="source-line-no">1799</span><span id="line-1799"> }</span>
<span class="source-line-no">1800</span><span id="line-1800"> }</span>
<span class="source-line-no">1801</span><span id="line-1801"> }</span>
<span class="source-line-no">1802</span><span id="line-1802"> }</span>
<span class="source-line-no">1803</span><span id="line-1803"></span>
<span class="source-line-no">1804</span><span id="line-1804"> class CheckRegionConsistencyWorkItem implements Callable&lt;Void&gt; {</span>
<span class="source-line-no">1805</span><span id="line-1805"> private final String key;</span>
<span class="source-line-no">1806</span><span id="line-1806"> private final HbckRegionInfo hbi;</span>
<span class="source-line-no">1807</span><span id="line-1807"></span>
<span class="source-line-no">1808</span><span id="line-1808"> CheckRegionConsistencyWorkItem(String key, HbckRegionInfo hbi) {</span>
<span class="source-line-no">1809</span><span id="line-1809"> this.key = key;</span>
<span class="source-line-no">1810</span><span id="line-1810"> this.hbi = hbi;</span>
<span class="source-line-no">1811</span><span id="line-1811"> }</span>
<span class="source-line-no">1812</span><span id="line-1812"></span>
<span class="source-line-no">1813</span><span id="line-1813"> @Override</span>
<span class="source-line-no">1814</span><span id="line-1814"> public synchronized Void call() throws Exception {</span>
<span class="source-line-no">1815</span><span id="line-1815"> try {</span>
<span class="source-line-no">1816</span><span id="line-1816"> checkRegionConsistency(key, hbi);</span>
<span class="source-line-no">1817</span><span id="line-1817"> } catch (Exception e) {</span>
<span class="source-line-no">1818</span><span id="line-1818"> // If the region is non-META region, skip this region and send warning/error message; if</span>
<span class="source-line-no">1819</span><span id="line-1819"> // the region is META region, we should not continue.</span>
<span class="source-line-no">1820</span><span id="line-1820"> LOG.warn(</span>
<span class="source-line-no">1821</span><span id="line-1821"> "Unable to complete check or repair the region '" + hbi.getRegionNameAsString() + "'.",</span>
<span class="source-line-no">1822</span><span id="line-1822"> e);</span>
<span class="source-line-no">1823</span><span id="line-1823"> if (hbi.getHdfsHRI().isMetaRegion()) {</span>
<span class="source-line-no">1824</span><span id="line-1824"> throw e;</span>
<span class="source-line-no">1825</span><span id="line-1825"> }</span>
<span class="source-line-no">1826</span><span id="line-1826"> LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'");</span>
<span class="source-line-no">1827</span><span id="line-1827"> addSkippedRegion(hbi);</span>
<span class="source-line-no">1828</span><span id="line-1828"> }</span>
<span class="source-line-no">1829</span><span id="line-1829"> return null;</span>
<span class="source-line-no">1830</span><span id="line-1830"> }</span>
<span class="source-line-no">1831</span><span id="line-1831"> }</span>
<span class="source-line-no">1832</span><span id="line-1832"></span>
<span class="source-line-no">1833</span><span id="line-1833"> private void addSkippedRegion(final HbckRegionInfo hbi) {</span>
<span class="source-line-no">1834</span><span id="line-1834"> Set&lt;String&gt; skippedRegionNames = skippedRegions.get(hbi.getTableName());</span>
<span class="source-line-no">1835</span><span id="line-1835"> if (skippedRegionNames == null) {</span>
<span class="source-line-no">1836</span><span id="line-1836"> skippedRegionNames = new HashSet&lt;&gt;();</span>
<span class="source-line-no">1837</span><span id="line-1837"> }</span>
<span class="source-line-no">1838</span><span id="line-1838"> skippedRegionNames.add(hbi.getRegionNameAsString());</span>
<span class="source-line-no">1839</span><span id="line-1839"> skippedRegions.put(hbi.getTableName(), skippedRegionNames);</span>
<span class="source-line-no">1840</span><span id="line-1840"> }</span>
<span class="source-line-no">1841</span><span id="line-1841"></span>
<span class="source-line-no">1842</span><span id="line-1842"> /**</span>
<span class="source-line-no">1843</span><span id="line-1843"> * Check and fix table states, assumes full info available: - tableInfos - empty tables loaded</span>
<span class="source-line-no">1844</span><span id="line-1844"> */</span>
<span class="source-line-no">1845</span><span id="line-1845"> private void checkAndFixTableStates() throws IOException {</span>
<span class="source-line-no">1846</span><span id="line-1846"> // first check dangling states</span>
<span class="source-line-no">1847</span><span id="line-1847"> for (Entry&lt;TableName, TableState&gt; entry : tableStates.entrySet()) {</span>
<span class="source-line-no">1848</span><span id="line-1848"> TableName tableName = entry.getKey();</span>
<span class="source-line-no">1849</span><span id="line-1849"> TableState tableState = entry.getValue();</span>
<span class="source-line-no">1850</span><span id="line-1850"> HbckTableInfo tableInfo = tablesInfo.get(tableName);</span>
<span class="source-line-no">1851</span><span id="line-1851"> if (isTableIncluded(tableName) &amp;&amp; !tableName.isSystemTable() &amp;&amp; tableInfo == null) {</span>
<span class="source-line-no">1852</span><span id="line-1852"> if (fixMeta) {</span>
<span class="source-line-no">1853</span><span id="line-1853"> MetaTableAccessor.deleteTableState(connection, tableName);</span>
<span class="source-line-no">1854</span><span id="line-1854"> TableState state = MetaTableAccessor.getTableState(connection, tableName);</span>
<span class="source-line-no">1855</span><span id="line-1855"> if (state != null) {</span>
<span class="source-line-no">1856</span><span id="line-1856"> errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,</span>
<span class="source-line-no">1857</span><span id="line-1857"> tableName + " unable to delete dangling table state " + tableState);</span>
<span class="source-line-no">1858</span><span id="line-1858"> }</span>
<span class="source-line-no">1859</span><span id="line-1859"> } else if (!checkMetaOnly) {</span>
<span class="source-line-no">1860</span><span id="line-1860"> // dangling table state in meta if checkMetaOnly is false. If checkMetaOnly is</span>
<span class="source-line-no">1861</span><span id="line-1861"> // true, tableInfo will be null as tablesInfo are not polulated for all tables from hdfs</span>
<span class="source-line-no">1862</span><span id="line-1862"> errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE,</span>
<span class="source-line-no">1863</span><span id="line-1863"> tableName + " has dangling table state " + tableState);</span>
<span class="source-line-no">1864</span><span id="line-1864"> }</span>
<span class="source-line-no">1865</span><span id="line-1865"> }</span>
<span class="source-line-no">1866</span><span id="line-1866"> }</span>
<span class="source-line-no">1867</span><span id="line-1867"> // check that all tables have states</span>
<span class="source-line-no">1868</span><span id="line-1868"> for (TableName tableName : tablesInfo.keySet()) {</span>
<span class="source-line-no">1869</span><span id="line-1869"> if (isTableIncluded(tableName) &amp;&amp; !tableStates.containsKey(tableName)) {</span>
<span class="source-line-no">1870</span><span id="line-1870"> if (fixMeta) {</span>
<span class="source-line-no">1871</span><span id="line-1871"> MetaTableAccessor.updateTableState(connection, tableName, TableState.State.ENABLED);</span>
<span class="source-line-no">1872</span><span id="line-1872"> TableState newState = MetaTableAccessor.getTableState(connection, tableName);</span>
<span class="source-line-no">1873</span><span id="line-1873"> if (newState == null) {</span>
<span class="source-line-no">1874</span><span id="line-1874"> errors.reportError(ERROR_CODE.NO_TABLE_STATE,</span>
<span class="source-line-no">1875</span><span id="line-1875"> "Unable to change state for table " + tableName + " in meta ");</span>
<span class="source-line-no">1876</span><span id="line-1876"> }</span>
<span class="source-line-no">1877</span><span id="line-1877"> } else {</span>
<span class="source-line-no">1878</span><span id="line-1878"> errors.reportError(ERROR_CODE.NO_TABLE_STATE, tableName + " has no state in meta ");</span>
<span class="source-line-no">1879</span><span id="line-1879"> }</span>
<span class="source-line-no">1880</span><span id="line-1880"> }</span>
<span class="source-line-no">1881</span><span id="line-1881"> }</span>
<span class="source-line-no">1882</span><span id="line-1882"> }</span>
<span class="source-line-no">1883</span><span id="line-1883"></span>
<span class="source-line-no">1884</span><span id="line-1884"> private void preCheckPermission() throws IOException {</span>
<span class="source-line-no">1885</span><span id="line-1885"> if (shouldIgnorePreCheckPermission()) {</span>
<span class="source-line-no">1886</span><span id="line-1886"> return;</span>
<span class="source-line-no">1887</span><span id="line-1887"> }</span>
<span class="source-line-no">1888</span><span id="line-1888"></span>
<span class="source-line-no">1889</span><span id="line-1889"> Path hbaseDir = CommonFSUtils.getRootDir(getConf());</span>
<span class="source-line-no">1890</span><span id="line-1890"> FileSystem fs = hbaseDir.getFileSystem(getConf());</span>
<span class="source-line-no">1891</span><span id="line-1891"> UserProvider userProvider = UserProvider.instantiate(getConf());</span>
<span class="source-line-no">1892</span><span id="line-1892"> UserGroupInformation ugi = userProvider.getCurrent().getUGI();</span>
<span class="source-line-no">1893</span><span id="line-1893"> FileStatus[] files = fs.listStatus(hbaseDir);</span>
<span class="source-line-no">1894</span><span id="line-1894"> for (FileStatus file : files) {</span>
<span class="source-line-no">1895</span><span id="line-1895"> try {</span>
<span class="source-line-no">1896</span><span id="line-1896"> fs.access(file.getPath(), FsAction.WRITE);</span>
<span class="source-line-no">1897</span><span id="line-1897"> } catch (AccessControlException ace) {</span>
<span class="source-line-no">1898</span><span id="line-1898"> LOG.warn("Got AccessDeniedException when preCheckPermission ", ace);</span>
<span class="source-line-no">1899</span><span id="line-1899"> errors.reportError(ERROR_CODE.WRONG_USAGE,</span>
<span class="source-line-no">1900</span><span id="line-1900"> "Current user " + ugi.getUserName() + " does not have write perms to " + file.getPath()</span>
<span class="source-line-no">1901</span><span id="line-1901"> + ". Please rerun hbck as hdfs user " + file.getOwner());</span>
<span class="source-line-no">1902</span><span id="line-1902"> throw ace;</span>
<span class="source-line-no">1903</span><span id="line-1903"> }</span>
<span class="source-line-no">1904</span><span id="line-1904"> }</span>
<span class="source-line-no">1905</span><span id="line-1905"> }</span>
<span class="source-line-no">1906</span><span id="line-1906"></span>
<span class="source-line-no">1907</span><span id="line-1907"> /**</span>
<span class="source-line-no">1908</span><span id="line-1908"> * Deletes region from meta table</span>
<span class="source-line-no">1909</span><span id="line-1909"> */</span>
<span class="source-line-no">1910</span><span id="line-1910"> private void deleteMetaRegion(HbckRegionInfo hi) throws IOException {</span>
<span class="source-line-no">1911</span><span id="line-1911"> deleteMetaRegion(hi.getMetaEntry().getRegionInfo().getRegionName());</span>
<span class="source-line-no">1912</span><span id="line-1912"> }</span>
<span class="source-line-no">1913</span><span id="line-1913"></span>
<span class="source-line-no">1914</span><span id="line-1914"> /**</span>
<span class="source-line-no">1915</span><span id="line-1915"> * Deletes region from meta table</span>
<span class="source-line-no">1916</span><span id="line-1916"> */</span>
<span class="source-line-no">1917</span><span id="line-1917"> private void deleteMetaRegion(byte[] metaKey) throws IOException {</span>
<span class="source-line-no">1918</span><span id="line-1918"> Delete d = new Delete(metaKey);</span>
<span class="source-line-no">1919</span><span id="line-1919"> meta.delete(d);</span>
<span class="source-line-no">1920</span><span id="line-1920"> LOG.info("Deleted " + Bytes.toString(metaKey) + " from META");</span>
<span class="source-line-no">1921</span><span id="line-1921"> }</span>
<span class="source-line-no">1922</span><span id="line-1922"></span>
<span class="source-line-no">1923</span><span id="line-1923"> /**</span>
<span class="source-line-no">1924</span><span id="line-1924"> * Reset the split parent region info in meta table</span>
<span class="source-line-no">1925</span><span id="line-1925"> */</span>
<span class="source-line-no">1926</span><span id="line-1926"> private void resetSplitParent(HbckRegionInfo hi) throws IOException {</span>
<span class="source-line-no">1927</span><span id="line-1927"> RowMutations mutations = new RowMutations(hi.getMetaEntry().getRegionInfo().getRegionName());</span>
<span class="source-line-no">1928</span><span id="line-1928"> Delete d = new Delete(hi.getMetaEntry().getRegionInfo().getRegionName());</span>
<span class="source-line-no">1929</span><span id="line-1929"> d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);</span>
<span class="source-line-no">1930</span><span id="line-1930"> d.addColumn(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);</span>
<span class="source-line-no">1931</span><span id="line-1931"> mutations.add(d);</span>
<span class="source-line-no">1932</span><span id="line-1932"></span>
<span class="source-line-no">1933</span><span id="line-1933"> RegionInfo hri = RegionInfoBuilder.newBuilder(hi.getMetaEntry().getRegionInfo())</span>
<span class="source-line-no">1934</span><span id="line-1934"> .setOffline(false).setSplit(false).build();</span>
<span class="source-line-no">1935</span><span id="line-1935"> Put p = MetaTableAccessor.makePutFromRegionInfo(hri);</span>
<span class="source-line-no">1936</span><span id="line-1936"> mutations.add(p);</span>
<span class="source-line-no">1937</span><span id="line-1937"></span>
<span class="source-line-no">1938</span><span id="line-1938"> meta.mutateRow(mutations);</span>
<span class="source-line-no">1939</span><span id="line-1939"> LOG.info("Reset split parent " + hi.getMetaEntry().getRegionInfo().getRegionNameAsString()</span>
<span class="source-line-no">1940</span><span id="line-1940"> + " in META");</span>
<span class="source-line-no">1941</span><span id="line-1941"> }</span>
<span class="source-line-no">1942</span><span id="line-1942"></span>
<span class="source-line-no">1943</span><span id="line-1943"> /**</span>
<span class="source-line-no">1944</span><span id="line-1944"> * This backwards-compatibility wrapper for permanently offlining a region that should not be</span>
<span class="source-line-no">1945</span><span id="line-1945"> * alive. If the region server does not support the "offline" method, it will use the closest</span>
<span class="source-line-no">1946</span><span id="line-1946"> * unassign method instead. This will basically work until one attempts to disable or delete the</span>
<span class="source-line-no">1947</span><span id="line-1947"> * affected table. The problem has to do with in-memory only master state, so restarting the</span>
<span class="source-line-no">1948</span><span id="line-1948"> * HMaster or failing over to another should fix this.</span>
<span class="source-line-no">1949</span><span id="line-1949"> */</span>
<span class="source-line-no">1950</span><span id="line-1950"> void offline(byte[] regionName) throws IOException {</span>
<span class="source-line-no">1951</span><span id="line-1951"> String regionString = Bytes.toStringBinary(regionName);</span>
<span class="source-line-no">1952</span><span id="line-1952"> if (!rsSupportsOffline) {</span>
<span class="source-line-no">1953</span><span id="line-1953"> LOG.warn("Using unassign region " + regionString</span>
<span class="source-line-no">1954</span><span id="line-1954"> + " instead of using offline method, you should" + " restart HMaster after these repairs");</span>
<span class="source-line-no">1955</span><span id="line-1955"> admin.unassign(regionName, true);</span>
<span class="source-line-no">1956</span><span id="line-1956"> return;</span>
<span class="source-line-no">1957</span><span id="line-1957"> }</span>
<span class="source-line-no">1958</span><span id="line-1958"></span>
<span class="source-line-no">1959</span><span id="line-1959"> // first time we assume the rs's supports #offline.</span>
<span class="source-line-no">1960</span><span id="line-1960"> try {</span>
<span class="source-line-no">1961</span><span id="line-1961"> LOG.info("Offlining region " + regionString);</span>
<span class="source-line-no">1962</span><span id="line-1962"> admin.offline(regionName);</span>
<span class="source-line-no">1963</span><span id="line-1963"> } catch (IOException ioe) {</span>
<span class="source-line-no">1964</span><span id="line-1964"> String notFoundMsg =</span>
<span class="source-line-no">1965</span><span id="line-1965"> "java.lang.NoSuchMethodException: " + "org.apache.hadoop.hbase.master.HMaster.offline([B)";</span>
<span class="source-line-no">1966</span><span id="line-1966"> if (ioe.getMessage().contains(notFoundMsg)) {</span>
<span class="source-line-no">1967</span><span id="line-1967"> LOG.warn(</span>
<span class="source-line-no">1968</span><span id="line-1968"> "Using unassign region " + regionString + " instead of using offline method, you should"</span>
<span class="source-line-no">1969</span><span id="line-1969"> + " restart HMaster after these repairs");</span>
<span class="source-line-no">1970</span><span id="line-1970"> rsSupportsOffline = false; // in the future just use unassign</span>
<span class="source-line-no">1971</span><span id="line-1971"> admin.unassign(regionName, true);</span>
<span class="source-line-no">1972</span><span id="line-1972"> return;</span>
<span class="source-line-no">1973</span><span id="line-1973"> }</span>
<span class="source-line-no">1974</span><span id="line-1974"> throw ioe;</span>
<span class="source-line-no">1975</span><span id="line-1975"> }</span>
<span class="source-line-no">1976</span><span id="line-1976"> }</span>
<span class="source-line-no">1977</span><span id="line-1977"></span>
<span class="source-line-no">1978</span><span id="line-1978"> /**</span>
<span class="source-line-no">1979</span><span id="line-1979"> * Attempts to undeploy a region from a region server based in information in META. Any operations</span>
<span class="source-line-no">1980</span><span id="line-1980"> * that modify the file system should make sure that its corresponding region is not deployed to</span>
<span class="source-line-no">1981</span><span id="line-1981"> * prevent data races. A separate call is required to update the master in-memory region state</span>
<span class="source-line-no">1982</span><span id="line-1982"> * kept in the AssignementManager. Because disable uses this state instead of that found in META,</span>
<span class="source-line-no">1983</span><span id="line-1983"> * we can't seem to cleanly disable/delete tables that have been hbck fixed. When used on a</span>
<span class="source-line-no">1984</span><span id="line-1984"> * version of HBase that does not have the offline ipc call exposed on the master (&amp;lt;0.90.5,</span>
<span class="source-line-no">1985</span><span id="line-1985"> * &amp;lt;0.92.0) a master restart or failover may be required.</span>
<span class="source-line-no">1986</span><span id="line-1986"> */</span>
<span class="source-line-no">1987</span><span id="line-1987"> void closeRegion(HbckRegionInfo hi) throws IOException, InterruptedException {</span>
<span class="source-line-no">1988</span><span id="line-1988"> if (hi.getMetaEntry() == null &amp;&amp; hi.getHdfsEntry() == null) {</span>
<span class="source-line-no">1989</span><span id="line-1989"> undeployRegions(hi);</span>
<span class="source-line-no">1990</span><span id="line-1990"> return;</span>
<span class="source-line-no">1991</span><span id="line-1991"> }</span>
<span class="source-line-no">1992</span><span id="line-1992"></span>
<span class="source-line-no">1993</span><span id="line-1993"> // get assignment info and hregioninfo from meta.</span>
<span class="source-line-no">1994</span><span id="line-1994"> Get get = new Get(hi.getRegionName());</span>
<span class="source-line-no">1995</span><span id="line-1995"> get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);</span>
<span class="source-line-no">1996</span><span id="line-1996"> get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);</span>
<span class="source-line-no">1997</span><span id="line-1997"> get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);</span>
<span class="source-line-no">1998</span><span id="line-1998"> // also get the locations of the replicas to close if the primary region is being closed</span>
<span class="source-line-no">1999</span><span id="line-1999"> if (hi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">2000</span><span id="line-2000"> int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication();</span>
<span class="source-line-no">2001</span><span id="line-2001"> for (int i = 0; i &lt; numReplicas; i++) {</span>
<span class="source-line-no">2002</span><span id="line-2002"> get.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(i));</span>
<span class="source-line-no">2003</span><span id="line-2003"> get.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(i));</span>
<span class="source-line-no">2004</span><span id="line-2004"> }</span>
<span class="source-line-no">2005</span><span id="line-2005"> }</span>
<span class="source-line-no">2006</span><span id="line-2006"> Result r = meta.get(get);</span>
<span class="source-line-no">2007</span><span id="line-2007"> RegionLocations rl = CatalogFamilyFormat.getRegionLocations(r);</span>
<span class="source-line-no">2008</span><span id="line-2008"> if (rl == null) {</span>
<span class="source-line-no">2009</span><span id="line-2009"> LOG.warn("Unable to close region " + hi.getRegionNameAsString()</span>
<span class="source-line-no">2010</span><span id="line-2010"> + " since meta does not have handle to reach it");</span>
<span class="source-line-no">2011</span><span id="line-2011"> return;</span>
<span class="source-line-no">2012</span><span id="line-2012"> }</span>
<span class="source-line-no">2013</span><span id="line-2013"> for (HRegionLocation h : rl.getRegionLocations()) {</span>
<span class="source-line-no">2014</span><span id="line-2014"> ServerName serverName = h.getServerName();</span>
<span class="source-line-no">2015</span><span id="line-2015"> if (serverName == null) {</span>
<span class="source-line-no">2016</span><span id="line-2016"> errors.reportError("Unable to close region " + hi.getRegionNameAsString()</span>
<span class="source-line-no">2017</span><span id="line-2017"> + " because meta does not " + "have handle to reach it.");</span>
<span class="source-line-no">2018</span><span id="line-2018"> continue;</span>
<span class="source-line-no">2019</span><span id="line-2019"> }</span>
<span class="source-line-no">2020</span><span id="line-2020"> RegionInfo hri = h.getRegion();</span>
<span class="source-line-no">2021</span><span id="line-2021"> if (hri == null) {</span>
<span class="source-line-no">2022</span><span id="line-2022"> LOG.warn("Unable to close region " + hi.getRegionNameAsString()</span>
<span class="source-line-no">2023</span><span id="line-2023"> + " because hbase:meta had invalid or missing " + HConstants.CATALOG_FAMILY_STR + ":"</span>
<span class="source-line-no">2024</span><span id="line-2024"> + Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " qualifier value.");</span>
<span class="source-line-no">2025</span><span id="line-2025"> continue;</span>
<span class="source-line-no">2026</span><span id="line-2026"> }</span>
<span class="source-line-no">2027</span><span id="line-2027"> // close the region -- close files and remove assignment</span>
<span class="source-line-no">2028</span><span id="line-2028"> HBaseFsckRepair.closeRegionSilentlyAndWait(connection, serverName, hri);</span>
<span class="source-line-no">2029</span><span id="line-2029"> }</span>
<span class="source-line-no">2030</span><span id="line-2030"> }</span>
<span class="source-line-no">2031</span><span id="line-2031"></span>
<span class="source-line-no">2032</span><span id="line-2032"> private void undeployRegions(HbckRegionInfo hi) throws IOException, InterruptedException {</span>
<span class="source-line-no">2033</span><span id="line-2033"> undeployRegionsForHbi(hi);</span>
<span class="source-line-no">2034</span><span id="line-2034"> // undeploy replicas of the region (but only if the method is invoked for the primary)</span>
<span class="source-line-no">2035</span><span id="line-2035"> if (hi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">2036</span><span id="line-2036"> return;</span>
<span class="source-line-no">2037</span><span id="line-2037"> }</span>
<span class="source-line-no">2038</span><span id="line-2038"> int numReplicas = admin.getDescriptor(hi.getTableName()).getRegionReplication();</span>
<span class="source-line-no">2039</span><span id="line-2039"> for (int i = 1; i &lt; numReplicas; i++) {</span>
<span class="source-line-no">2040</span><span id="line-2040"> if (hi.getPrimaryHRIForDeployedReplica() == null) continue;</span>
<span class="source-line-no">2041</span><span id="line-2041"> RegionInfo hri =</span>
<span class="source-line-no">2042</span><span id="line-2042"> RegionReplicaUtil.getRegionInfoForReplica(hi.getPrimaryHRIForDeployedReplica(), i);</span>
<span class="source-line-no">2043</span><span id="line-2043"> HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());</span>
<span class="source-line-no">2044</span><span id="line-2044"> if (h != null) {</span>
<span class="source-line-no">2045</span><span id="line-2045"> undeployRegionsForHbi(h);</span>
<span class="source-line-no">2046</span><span id="line-2046"> // set skip checks; we undeployed it, and we don't want to evaluate this anymore</span>
<span class="source-line-no">2047</span><span id="line-2047"> // in consistency checks</span>
<span class="source-line-no">2048</span><span id="line-2048"> h.setSkipChecks(true);</span>
<span class="source-line-no">2049</span><span id="line-2049"> }</span>
<span class="source-line-no">2050</span><span id="line-2050"> }</span>
<span class="source-line-no">2051</span><span id="line-2051"> }</span>
<span class="source-line-no">2052</span><span id="line-2052"></span>
<span class="source-line-no">2053</span><span id="line-2053"> private void undeployRegionsForHbi(HbckRegionInfo hi) throws IOException, InterruptedException {</span>
<span class="source-line-no">2054</span><span id="line-2054"> for (HbckRegionInfo.OnlineEntry rse : hi.getOnlineEntries()) {</span>
<span class="source-line-no">2055</span><span id="line-2055"> LOG.debug("Undeploy region " + rse.getRegionInfo() + " from " + rse.getServerName());</span>
<span class="source-line-no">2056</span><span id="line-2056"> try {</span>
<span class="source-line-no">2057</span><span id="line-2057"> HBaseFsckRepair.closeRegionSilentlyAndWait(connection, rse.getServerName(),</span>
<span class="source-line-no">2058</span><span id="line-2058"> rse.getRegionInfo());</span>
<span class="source-line-no">2059</span><span id="line-2059"> offline(rse.getRegionInfo().getRegionName());</span>
<span class="source-line-no">2060</span><span id="line-2060"> } catch (IOException ioe) {</span>
<span class="source-line-no">2061</span><span id="line-2061"> LOG.warn("Got exception when attempting to offline region "</span>
<span class="source-line-no">2062</span><span id="line-2062"> + Bytes.toString(rse.getRegionInfo().getRegionName()), ioe);</span>
<span class="source-line-no">2063</span><span id="line-2063"> }</span>
<span class="source-line-no">2064</span><span id="line-2064"> }</span>
<span class="source-line-no">2065</span><span id="line-2065"> }</span>
<span class="source-line-no">2066</span><span id="line-2066"></span>
<span class="source-line-no">2067</span><span id="line-2067"> private void tryAssignmentRepair(HbckRegionInfo hbi, String msg)</span>
<span class="source-line-no">2068</span><span id="line-2068"> throws IOException, KeeperException, InterruptedException {</span>
<span class="source-line-no">2069</span><span id="line-2069"> // If we are trying to fix the errors</span>
<span class="source-line-no">2070</span><span id="line-2070"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2071</span><span id="line-2071"> errors.print(msg);</span>
<span class="source-line-no">2072</span><span id="line-2072"> undeployRegions(hbi);</span>
<span class="source-line-no">2073</span><span id="line-2073"> setShouldRerun();</span>
<span class="source-line-no">2074</span><span id="line-2074"> RegionInfo hri = hbi.getHdfsHRI();</span>
<span class="source-line-no">2075</span><span id="line-2075"> if (hri == null) {</span>
<span class="source-line-no">2076</span><span id="line-2076"> hri = hbi.getMetaEntry().getRegionInfo();</span>
<span class="source-line-no">2077</span><span id="line-2077"> }</span>
<span class="source-line-no">2078</span><span id="line-2078"> HBaseFsckRepair.fixUnassigned(admin, hri);</span>
<span class="source-line-no">2079</span><span id="line-2079"> HBaseFsckRepair.waitUntilAssigned(admin, hri);</span>
<span class="source-line-no">2080</span><span id="line-2080"></span>
<span class="source-line-no">2081</span><span id="line-2081"> // also assign replicas if needed (do it only when this call operates on a primary replica)</span>
<span class="source-line-no">2082</span><span id="line-2082"> if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) return;</span>
<span class="source-line-no">2083</span><span id="line-2083"> int replicationCount = admin.getDescriptor(hri.getTable()).getRegionReplication();</span>
<span class="source-line-no">2084</span><span id="line-2084"> for (int i = 1; i &lt; replicationCount; i++) {</span>
<span class="source-line-no">2085</span><span id="line-2085"> hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i);</span>
<span class="source-line-no">2086</span><span id="line-2086"> HbckRegionInfo h = regionInfoMap.get(hri.getEncodedName());</span>
<span class="source-line-no">2087</span><span id="line-2087"> if (h != null) {</span>
<span class="source-line-no">2088</span><span id="line-2088"> undeployRegions(h);</span>
<span class="source-line-no">2089</span><span id="line-2089"> // set skip checks; we undeploy &amp; deploy it; we don't want to evaluate this hbi anymore</span>
<span class="source-line-no">2090</span><span id="line-2090"> // in consistency checks</span>
<span class="source-line-no">2091</span><span id="line-2091"> h.setSkipChecks(true);</span>
<span class="source-line-no">2092</span><span id="line-2092"> }</span>
<span class="source-line-no">2093</span><span id="line-2093"> HBaseFsckRepair.fixUnassigned(admin, hri);</span>
<span class="source-line-no">2094</span><span id="line-2094"> HBaseFsckRepair.waitUntilAssigned(admin, hri);</span>
<span class="source-line-no">2095</span><span id="line-2095"> }</span>
<span class="source-line-no">2096</span><span id="line-2096"></span>
<span class="source-line-no">2097</span><span id="line-2097"> }</span>
<span class="source-line-no">2098</span><span id="line-2098"> }</span>
<span class="source-line-no">2099</span><span id="line-2099"></span>
<span class="source-line-no">2100</span><span id="line-2100"> /**</span>
<span class="source-line-no">2101</span><span id="line-2101"> * Check a single region for consistency and correct deployment.</span>
<span class="source-line-no">2102</span><span id="line-2102"> */</span>
<span class="source-line-no">2103</span><span id="line-2103"> private void checkRegionConsistency(final String key, final HbckRegionInfo hbi)</span>
<span class="source-line-no">2104</span><span id="line-2104"> throws IOException, KeeperException, InterruptedException {</span>
<span class="source-line-no">2105</span><span id="line-2105"></span>
<span class="source-line-no">2106</span><span id="line-2106"> if (hbi.isSkipChecks()) return;</span>
<span class="source-line-no">2107</span><span id="line-2107"> String descriptiveName = hbi.toString();</span>
<span class="source-line-no">2108</span><span id="line-2108"> boolean inMeta = hbi.getMetaEntry() != null;</span>
<span class="source-line-no">2109</span><span id="line-2109"> // In case not checking HDFS, assume the region is on HDFS</span>
<span class="source-line-no">2110</span><span id="line-2110"> boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;</span>
<span class="source-line-no">2111</span><span id="line-2111"> boolean hasMetaAssignment = inMeta &amp;&amp; hbi.getMetaEntry().regionServer != null;</span>
<span class="source-line-no">2112</span><span id="line-2112"> boolean isDeployed = !hbi.getDeployedOn().isEmpty();</span>
<span class="source-line-no">2113</span><span id="line-2113"> boolean isMultiplyDeployed = hbi.getDeployedOn().size() &gt; 1;</span>
<span class="source-line-no">2114</span><span id="line-2114"> boolean deploymentMatchesMeta = hasMetaAssignment &amp;&amp; isDeployed &amp;&amp; !isMultiplyDeployed</span>
<span class="source-line-no">2115</span><span id="line-2115"> &amp;&amp; hbi.getMetaEntry().regionServer.equals(hbi.getDeployedOn().get(0));</span>
<span class="source-line-no">2116</span><span id="line-2116"> boolean splitParent = inMeta &amp;&amp; hbi.getMetaEntry().getRegionInfo().isSplit()</span>
<span class="source-line-no">2117</span><span id="line-2117"> &amp;&amp; hbi.getMetaEntry().getRegionInfo().isOffline();</span>
<span class="source-line-no">2118</span><span id="line-2118"> boolean shouldBeDeployed =</span>
<span class="source-line-no">2119</span><span id="line-2119"> inMeta &amp;&amp; !isTableDisabled(hbi.getMetaEntry().getRegionInfo().getTable());</span>
<span class="source-line-no">2120</span><span id="line-2120"> boolean recentlyModified =</span>
<span class="source-line-no">2121</span><span id="line-2121"> inHdfs &amp;&amp; hbi.getModTime() + timelag &gt; EnvironmentEdgeManager.currentTime();</span>
<span class="source-line-no">2122</span><span id="line-2122"></span>
<span class="source-line-no">2123</span><span id="line-2123"> // ========== First the healthy cases =============</span>
<span class="source-line-no">2124</span><span id="line-2124"> if (hbi.containsOnlyHdfsEdits()) {</span>
<span class="source-line-no">2125</span><span id="line-2125"> return;</span>
<span class="source-line-no">2126</span><span id="line-2126"> }</span>
<span class="source-line-no">2127</span><span id="line-2127"> if (inMeta &amp;&amp; inHdfs &amp;&amp; isDeployed &amp;&amp; deploymentMatchesMeta &amp;&amp; shouldBeDeployed) {</span>
<span class="source-line-no">2128</span><span id="line-2128"> return;</span>
<span class="source-line-no">2129</span><span id="line-2129"> } else if (inMeta &amp;&amp; inHdfs &amp;&amp; !shouldBeDeployed &amp;&amp; !isDeployed) {</span>
<span class="source-line-no">2130</span><span id="line-2130"> LOG.info("Region " + descriptiveName + " is in META, and in a disabled "</span>
<span class="source-line-no">2131</span><span id="line-2131"> + "tabled that is not deployed");</span>
<span class="source-line-no">2132</span><span id="line-2132"> return;</span>
<span class="source-line-no">2133</span><span id="line-2133"> } else if (recentlyModified) {</span>
<span class="source-line-no">2134</span><span id="line-2134"> LOG.warn("Region " + descriptiveName + " was recently modified -- skipping");</span>
<span class="source-line-no">2135</span><span id="line-2135"> return;</span>
<span class="source-line-no">2136</span><span id="line-2136"> }</span>
<span class="source-line-no">2137</span><span id="line-2137"> // ========== Cases where the region is not in hbase:meta =============</span>
<span class="source-line-no">2138</span><span id="line-2138"> else if (!inMeta &amp;&amp; !inHdfs &amp;&amp; !isDeployed) {</span>
<span class="source-line-no">2139</span><span id="line-2139"> // We shouldn't have record of this region at all then!</span>
<span class="source-line-no">2140</span><span id="line-2140"> assert false : "Entry for region with no data";</span>
<span class="source-line-no">2141</span><span id="line-2141"> } else if (!inMeta &amp;&amp; !inHdfs &amp;&amp; isDeployed) {</span>
<span class="source-line-no">2142</span><span id="line-2142"> errors.reportError(ERROR_CODE.NOT_IN_META_HDFS,</span>
<span class="source-line-no">2143</span><span id="line-2143"> "Region " + descriptiveName + ", key=" + key + ", not on HDFS or in hbase:meta but "</span>
<span class="source-line-no">2144</span><span id="line-2144"> + "deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));</span>
<span class="source-line-no">2145</span><span id="line-2145"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2146</span><span id="line-2146"> undeployRegions(hbi);</span>
<span class="source-line-no">2147</span><span id="line-2147"> }</span>
<span class="source-line-no">2148</span><span id="line-2148"></span>
<span class="source-line-no">2149</span><span id="line-2149"> } else if (!inMeta &amp;&amp; inHdfs &amp;&amp; !isDeployed) {</span>
<span class="source-line-no">2150</span><span id="line-2150"> if (hbi.isMerged()) {</span>
<span class="source-line-no">2151</span><span id="line-2151"> // This region has already been merged, the remaining hdfs file will be</span>
<span class="source-line-no">2152</span><span id="line-2152"> // cleaned by CatalogJanitor later</span>
<span class="source-line-no">2153</span><span id="line-2153"> hbi.setSkipChecks(true);</span>
<span class="source-line-no">2154</span><span id="line-2154"> LOG.info("Region " + descriptiveName</span>
<span class="source-line-no">2155</span><span id="line-2155"> + " got merge recently, its file(s) will be cleaned by CatalogJanitor later");</span>
<span class="source-line-no">2156</span><span id="line-2156"> return;</span>
<span class="source-line-no">2157</span><span id="line-2157"> }</span>
<span class="source-line-no">2158</span><span id="line-2158"> errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region " + descriptiveName</span>
<span class="source-line-no">2159</span><span id="line-2159"> + " on HDFS, but not listed in hbase:meta " + "or deployed on any region server");</span>
<span class="source-line-no">2160</span><span id="line-2160"> // restore region consistency of an adopted orphan</span>
<span class="source-line-no">2161</span><span id="line-2161"> if (shouldFixMeta()) {</span>
<span class="source-line-no">2162</span><span id="line-2162"> if (!hbi.isHdfsRegioninfoPresent()) {</span>
<span class="source-line-no">2163</span><span id="line-2163"> LOG.error("Region " + hbi.getHdfsHRI() + " could have been repaired"</span>
<span class="source-line-no">2164</span><span id="line-2164"> + " in table integrity repair phase if -fixHdfsOrphans was" + " used.");</span>
<span class="source-line-no">2165</span><span id="line-2165"> return;</span>
<span class="source-line-no">2166</span><span id="line-2166"> }</span>
<span class="source-line-no">2167</span><span id="line-2167"></span>
<span class="source-line-no">2168</span><span id="line-2168"> RegionInfo hri = hbi.getHdfsHRI();</span>
<span class="source-line-no">2169</span><span id="line-2169"> HbckTableInfo tableInfo = tablesInfo.get(hri.getTable());</span>
<span class="source-line-no">2170</span><span id="line-2170"></span>
<span class="source-line-no">2171</span><span id="line-2171"> for (RegionInfo region : tableInfo.getRegionsFromMeta(this.regionInfoMap)) {</span>
<span class="source-line-no">2172</span><span id="line-2172"> if (</span>
<span class="source-line-no">2173</span><span id="line-2173"> Bytes.compareTo(region.getStartKey(), hri.getStartKey()) &lt;= 0</span>
<span class="source-line-no">2174</span><span id="line-2174"> &amp;&amp; (region.getEndKey().length == 0</span>
<span class="source-line-no">2175</span><span id="line-2175"> || Bytes.compareTo(region.getEndKey(), hri.getEndKey()) &gt;= 0)</span>
<span class="source-line-no">2176</span><span id="line-2176"> &amp;&amp; Bytes.compareTo(region.getStartKey(), hri.getEndKey()) &lt;= 0</span>
<span class="source-line-no">2177</span><span id="line-2177"> ) {</span>
<span class="source-line-no">2178</span><span id="line-2178"> if (region.isSplit() || region.isOffline()) continue;</span>
<span class="source-line-no">2179</span><span id="line-2179"> Path regionDir = hbi.getHdfsRegionDir();</span>
<span class="source-line-no">2180</span><span id="line-2180"> FileSystem fs = regionDir.getFileSystem(getConf());</span>
<span class="source-line-no">2181</span><span id="line-2181"> List&lt;Path&gt; familyDirs = FSUtils.getFamilyDirs(fs, regionDir);</span>
<span class="source-line-no">2182</span><span id="line-2182"> for (Path familyDir : familyDirs) {</span>
<span class="source-line-no">2183</span><span id="line-2183"> List&lt;Path&gt; referenceFilePaths = FSUtils.getReferenceFilePaths(fs, familyDir);</span>
<span class="source-line-no">2184</span><span id="line-2184"> for (Path referenceFilePath : referenceFilePaths) {</span>
<span class="source-line-no">2185</span><span id="line-2185"> Path parentRegionDir =</span>
<span class="source-line-no">2186</span><span id="line-2186"> StoreFileInfo.getReferredToFile(referenceFilePath).getParent().getParent();</span>
<span class="source-line-no">2187</span><span id="line-2187"> if (parentRegionDir.toString().endsWith(region.getEncodedName())) {</span>
<span class="source-line-no">2188</span><span id="line-2188"> LOG.warn(hri + " start and stop keys are in the range of " + region</span>
<span class="source-line-no">2189</span><span id="line-2189"> + ". The region might not be cleaned up from hdfs when region " + region</span>
<span class="source-line-no">2190</span><span id="line-2190"> + " split failed. Hence deleting from hdfs.");</span>
<span class="source-line-no">2191</span><span id="line-2191"> HRegionFileSystem.deleteRegionFromFileSystem(getConf(), fs, regionDir.getParent(),</span>
<span class="source-line-no">2192</span><span id="line-2192"> hri);</span>
<span class="source-line-no">2193</span><span id="line-2193"> return;</span>
<span class="source-line-no">2194</span><span id="line-2194"> }</span>
<span class="source-line-no">2195</span><span id="line-2195"> }</span>
<span class="source-line-no">2196</span><span id="line-2196"> }</span>
<span class="source-line-no">2197</span><span id="line-2197"> }</span>
<span class="source-line-no">2198</span><span id="line-2198"> }</span>
<span class="source-line-no">2199</span><span id="line-2199"> LOG.info("Patching hbase:meta with .regioninfo: " + hbi.getHdfsHRI());</span>
<span class="source-line-no">2200</span><span id="line-2200"> int numReplicas = admin.getDescriptor(hbi.getTableName()).getRegionReplication();</span>
<span class="source-line-no">2201</span><span id="line-2201"> HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),</span>
<span class="source-line-no">2202</span><span id="line-2202"> admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet(),</span>
<span class="source-line-no">2203</span><span id="line-2203"> numReplicas);</span>
<span class="source-line-no">2204</span><span id="line-2204"></span>
<span class="source-line-no">2205</span><span id="line-2205"> tryAssignmentRepair(hbi, "Trying to reassign region...");</span>
<span class="source-line-no">2206</span><span id="line-2206"> }</span>
<span class="source-line-no">2207</span><span id="line-2207"></span>
<span class="source-line-no">2208</span><span id="line-2208"> } else if (!inMeta &amp;&amp; inHdfs &amp;&amp; isDeployed) {</span>
<span class="source-line-no">2209</span><span id="line-2209"> errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName</span>
<span class="source-line-no">2210</span><span id="line-2210"> + " not in META, but deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));</span>
<span class="source-line-no">2211</span><span id="line-2211"> debugLsr(hbi.getHdfsRegionDir());</span>
<span class="source-line-no">2212</span><span id="line-2212"> if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">2213</span><span id="line-2213"> // for replicas, this means that we should undeploy the region (we would have</span>
<span class="source-line-no">2214</span><span id="line-2214"> // gone over the primaries and fixed meta holes in first phase under</span>
<span class="source-line-no">2215</span><span id="line-2215"> // checkAndFixConsistency; we shouldn't get the condition !inMeta at</span>
<span class="source-line-no">2216</span><span id="line-2216"> // this stage unless unwanted replica)</span>
<span class="source-line-no">2217</span><span id="line-2217"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2218</span><span id="line-2218"> undeployRegionsForHbi(hbi);</span>
<span class="source-line-no">2219</span><span id="line-2219"> }</span>
<span class="source-line-no">2220</span><span id="line-2220"> }</span>
<span class="source-line-no">2221</span><span id="line-2221"> if (shouldFixMeta() &amp;&amp; hbi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">2222</span><span id="line-2222"> if (!hbi.isHdfsRegioninfoPresent()) {</span>
<span class="source-line-no">2223</span><span id="line-2223"> LOG.error("This should have been repaired in table integrity repair phase");</span>
<span class="source-line-no">2224</span><span id="line-2224"> return;</span>
<span class="source-line-no">2225</span><span id="line-2225"> }</span>
<span class="source-line-no">2226</span><span id="line-2226"></span>
<span class="source-line-no">2227</span><span id="line-2227"> LOG.info("Patching hbase:meta with with .regioninfo: " + hbi.getHdfsHRI());</span>
<span class="source-line-no">2228</span><span id="line-2228"> int numReplicas = admin.getDescriptor(hbi.getTableName()).getRegionReplication();</span>
<span class="source-line-no">2229</span><span id="line-2229"> HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(),</span>
<span class="source-line-no">2230</span><span id="line-2230"> admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet(),</span>
<span class="source-line-no">2231</span><span id="line-2231"> numReplicas);</span>
<span class="source-line-no">2232</span><span id="line-2232"> tryAssignmentRepair(hbi, "Trying to fix unassigned region...");</span>
<span class="source-line-no">2233</span><span id="line-2233"> }</span>
<span class="source-line-no">2234</span><span id="line-2234"></span>
<span class="source-line-no">2235</span><span id="line-2235"> // ========== Cases where the region is in hbase:meta =============</span>
<span class="source-line-no">2236</span><span id="line-2236"> } else if (inMeta &amp;&amp; inHdfs &amp;&amp; !isDeployed &amp;&amp; splitParent) {</span>
<span class="source-line-no">2237</span><span id="line-2237"> // check whether this is an actual error, or just transient state where parent</span>
<span class="source-line-no">2238</span><span id="line-2238"> // is not cleaned</span>
<span class="source-line-no">2239</span><span id="line-2239"> if (hbi.getMetaEntry().splitA != null &amp;&amp; hbi.getMetaEntry().splitB != null) {</span>
<span class="source-line-no">2240</span><span id="line-2240"> // check that split daughters are there</span>
<span class="source-line-no">2241</span><span id="line-2241"> HbckRegionInfo infoA = this.regionInfoMap.get(hbi.getMetaEntry().splitA.getEncodedName());</span>
<span class="source-line-no">2242</span><span id="line-2242"> HbckRegionInfo infoB = this.regionInfoMap.get(hbi.getMetaEntry().splitB.getEncodedName());</span>
<span class="source-line-no">2243</span><span id="line-2243"> if (infoA != null &amp;&amp; infoB != null) {</span>
<span class="source-line-no">2244</span><span id="line-2244"> // we already processed or will process daughters. Move on, nothing to see here.</span>
<span class="source-line-no">2245</span><span id="line-2245"> hbi.setSkipChecks(true);</span>
<span class="source-line-no">2246</span><span id="line-2246"> return;</span>
<span class="source-line-no">2247</span><span id="line-2247"> }</span>
<span class="source-line-no">2248</span><span id="line-2248"> }</span>
<span class="source-line-no">2249</span><span id="line-2249"></span>
<span class="source-line-no">2250</span><span id="line-2250"> // For Replica region, we need to do a similar check. If replica is not split successfully,</span>
<span class="source-line-no">2251</span><span id="line-2251"> // error is going to be reported against primary daughter region.</span>
<span class="source-line-no">2252</span><span id="line-2252"> if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">2253</span><span id="line-2253"> LOG.info("Region " + descriptiveName + " is a split parent in META, in HDFS, "</span>
<span class="source-line-no">2254</span><span id="line-2254"> + "and not deployed on any region server. This may be transient.");</span>
<span class="source-line-no">2255</span><span id="line-2255"> hbi.setSkipChecks(true);</span>
<span class="source-line-no">2256</span><span id="line-2256"> return;</span>
<span class="source-line-no">2257</span><span id="line-2257"> }</span>
<span class="source-line-no">2258</span><span id="line-2258"></span>
<span class="source-line-no">2259</span><span id="line-2259"> errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT,</span>
<span class="source-line-no">2260</span><span id="line-2260"> "Region " + descriptiveName + " is a split parent in META, in HDFS, "</span>
<span class="source-line-no">2261</span><span id="line-2261"> + "and not deployed on any region server. This could be transient, "</span>
<span class="source-line-no">2262</span><span id="line-2262"> + "consider to run the catalog janitor first!");</span>
<span class="source-line-no">2263</span><span id="line-2263"> if (shouldFixSplitParents()) {</span>
<span class="source-line-no">2264</span><span id="line-2264"> setShouldRerun();</span>
<span class="source-line-no">2265</span><span id="line-2265"> resetSplitParent(hbi);</span>
<span class="source-line-no">2266</span><span id="line-2266"> }</span>
<span class="source-line-no">2267</span><span id="line-2267"> } else if (inMeta &amp;&amp; !inHdfs &amp;&amp; !isDeployed) {</span>
<span class="source-line-no">2268</span><span id="line-2268"> errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region " + descriptiveName</span>
<span class="source-line-no">2269</span><span id="line-2269"> + " found in META, but not in HDFS " + "or deployed on any region server.");</span>
<span class="source-line-no">2270</span><span id="line-2270"> if (shouldFixMeta()) {</span>
<span class="source-line-no">2271</span><span id="line-2271"> deleteMetaRegion(hbi);</span>
<span class="source-line-no">2272</span><span id="line-2272"> }</span>
<span class="source-line-no">2273</span><span id="line-2273"> } else if (inMeta &amp;&amp; !inHdfs &amp;&amp; isDeployed) {</span>
<span class="source-line-no">2274</span><span id="line-2274"> errors.reportError(ERROR_CODE.NOT_IN_HDFS,</span>
<span class="source-line-no">2275</span><span id="line-2275"> "Region " + descriptiveName + " found in META, but not in HDFS, " + "and deployed on "</span>
<span class="source-line-no">2276</span><span id="line-2276"> + Joiner.on(", ").join(hbi.getDeployedOn()));</span>
<span class="source-line-no">2277</span><span id="line-2277"> // We treat HDFS as ground truth. Any information in meta is transient</span>
<span class="source-line-no">2278</span><span id="line-2278"> // and equivalent data can be regenerated. So, lets unassign and remove</span>
<span class="source-line-no">2279</span><span id="line-2279"> // these problems from META.</span>
<span class="source-line-no">2280</span><span id="line-2280"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2281</span><span id="line-2281"> errors.print("Trying to fix unassigned region...");</span>
<span class="source-line-no">2282</span><span id="line-2282"> undeployRegions(hbi);</span>
<span class="source-line-no">2283</span><span id="line-2283"> }</span>
<span class="source-line-no">2284</span><span id="line-2284"> if (shouldFixMeta()) {</span>
<span class="source-line-no">2285</span><span id="line-2285"> // wait for it to complete</span>
<span class="source-line-no">2286</span><span id="line-2286"> deleteMetaRegion(hbi);</span>
<span class="source-line-no">2287</span><span id="line-2287"> }</span>
<span class="source-line-no">2288</span><span id="line-2288"> } else if (inMeta &amp;&amp; inHdfs &amp;&amp; !isDeployed &amp;&amp; shouldBeDeployed) {</span>
<span class="source-line-no">2289</span><span id="line-2289"> errors.reportError(ERROR_CODE.NOT_DEPLOYED,</span>
<span class="source-line-no">2290</span><span id="line-2290"> "Region " + descriptiveName + " not deployed on any region server.");</span>
<span class="source-line-no">2291</span><span id="line-2291"> tryAssignmentRepair(hbi, "Trying to fix unassigned region...");</span>
<span class="source-line-no">2292</span><span id="line-2292"> } else if (inMeta &amp;&amp; inHdfs &amp;&amp; isDeployed &amp;&amp; !shouldBeDeployed) {</span>
<span class="source-line-no">2293</span><span id="line-2293"> errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,</span>
<span class="source-line-no">2294</span><span id="line-2294"> "Region " + descriptiveName + " should not be deployed according "</span>
<span class="source-line-no">2295</span><span id="line-2295"> + "to META, but is deployed on " + Joiner.on(", ").join(hbi.getDeployedOn()));</span>
<span class="source-line-no">2296</span><span id="line-2296"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2297</span><span id="line-2297"> errors.print("Trying to close the region " + descriptiveName);</span>
<span class="source-line-no">2298</span><span id="line-2298"> setShouldRerun();</span>
<span class="source-line-no">2299</span><span id="line-2299"> HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry().getRegionInfo(),</span>
<span class="source-line-no">2300</span><span id="line-2300"> hbi.getDeployedOn());</span>
<span class="source-line-no">2301</span><span id="line-2301"> }</span>
<span class="source-line-no">2302</span><span id="line-2302"> } else if (inMeta &amp;&amp; inHdfs &amp;&amp; isMultiplyDeployed) {</span>
<span class="source-line-no">2303</span><span id="line-2303"> errors.reportError(ERROR_CODE.MULTI_DEPLOYED,</span>
<span class="source-line-no">2304</span><span id="line-2304"> "Region " + descriptiveName + " is listed in hbase:meta on region server "</span>
<span class="source-line-no">2305</span><span id="line-2305"> + hbi.getMetaEntry().regionServer + " but is multiply assigned to region servers "</span>
<span class="source-line-no">2306</span><span id="line-2306"> + Joiner.on(", ").join(hbi.getDeployedOn()));</span>
<span class="source-line-no">2307</span><span id="line-2307"> // If we are trying to fix the errors</span>
<span class="source-line-no">2308</span><span id="line-2308"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2309</span><span id="line-2309"> errors.print("Trying to fix assignment error...");</span>
<span class="source-line-no">2310</span><span id="line-2310"> setShouldRerun();</span>
<span class="source-line-no">2311</span><span id="line-2311"> HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry().getRegionInfo(),</span>
<span class="source-line-no">2312</span><span id="line-2312"> hbi.getDeployedOn());</span>
<span class="source-line-no">2313</span><span id="line-2313"> }</span>
<span class="source-line-no">2314</span><span id="line-2314"> } else if (inMeta &amp;&amp; inHdfs &amp;&amp; isDeployed &amp;&amp; !deploymentMatchesMeta) {</span>
<span class="source-line-no">2315</span><span id="line-2315"> errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META,</span>
<span class="source-line-no">2316</span><span id="line-2316"> "Region " + descriptiveName + " listed in hbase:meta on region server "</span>
<span class="source-line-no">2317</span><span id="line-2317"> + hbi.getMetaEntry().regionServer + " but found on region server "</span>
<span class="source-line-no">2318</span><span id="line-2318"> + hbi.getDeployedOn().get(0));</span>
<span class="source-line-no">2319</span><span id="line-2319"> // If we are trying to fix the errors</span>
<span class="source-line-no">2320</span><span id="line-2320"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2321</span><span id="line-2321"> errors.print("Trying to fix assignment error...");</span>
<span class="source-line-no">2322</span><span id="line-2322"> setShouldRerun();</span>
<span class="source-line-no">2323</span><span id="line-2323"> HBaseFsckRepair.fixMultiAssignment(connection, hbi.getMetaEntry().getRegionInfo(),</span>
<span class="source-line-no">2324</span><span id="line-2324"> hbi.getDeployedOn());</span>
<span class="source-line-no">2325</span><span id="line-2325"> HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());</span>
<span class="source-line-no">2326</span><span id="line-2326"> }</span>
<span class="source-line-no">2327</span><span id="line-2327"> } else {</span>
<span class="source-line-no">2328</span><span id="line-2328"> errors.reportError(ERROR_CODE.UNKNOWN,</span>
<span class="source-line-no">2329</span><span id="line-2329"> "Region " + descriptiveName + " is in an unforeseen state:" + " inMeta=" + inMeta</span>
<span class="source-line-no">2330</span><span id="line-2330"> + " inHdfs=" + inHdfs + " isDeployed=" + isDeployed + " isMultiplyDeployed="</span>
<span class="source-line-no">2331</span><span id="line-2331"> + isMultiplyDeployed + " deploymentMatchesMeta=" + deploymentMatchesMeta</span>
<span class="source-line-no">2332</span><span id="line-2332"> + " shouldBeDeployed=" + shouldBeDeployed);</span>
<span class="source-line-no">2333</span><span id="line-2333"> }</span>
<span class="source-line-no">2334</span><span id="line-2334"> }</span>
<span class="source-line-no">2335</span><span id="line-2335"></span>
<span class="source-line-no">2336</span><span id="line-2336"> /**</span>
<span class="source-line-no">2337</span><span id="line-2337"> * Checks tables integrity. Goes over all regions and scans the tables. Collects all the pieces</span>
<span class="source-line-no">2338</span><span id="line-2338"> * for each table and checks if there are missing, repeated or overlapping ones.</span>
<span class="source-line-no">2339</span><span id="line-2339"> */</span>
<span class="source-line-no">2340</span><span id="line-2340"> SortedMap&lt;TableName, HbckTableInfo&gt; checkIntegrity() throws IOException {</span>
<span class="source-line-no">2341</span><span id="line-2341"> tablesInfo = new TreeMap&lt;&gt;();</span>
<span class="source-line-no">2342</span><span id="line-2342"> LOG.debug("There are " + regionInfoMap.size() + " region info entries");</span>
<span class="source-line-no">2343</span><span id="line-2343"> for (HbckRegionInfo hbi : regionInfoMap.values()) {</span>
<span class="source-line-no">2344</span><span id="line-2344"> // Check only valid, working regions</span>
<span class="source-line-no">2345</span><span id="line-2345"> if (hbi.getMetaEntry() == null) {</span>
<span class="source-line-no">2346</span><span id="line-2346"> // this assumes that consistency check has run loadMetaEntry</span>
<span class="source-line-no">2347</span><span id="line-2347"> Path p = hbi.getHdfsRegionDir();</span>
<span class="source-line-no">2348</span><span id="line-2348"> if (p == null) {</span>
<span class="source-line-no">2349</span><span id="line-2349"> errors.report("No regioninfo in Meta or HDFS. " + hbi);</span>
<span class="source-line-no">2350</span><span id="line-2350"> }</span>
<span class="source-line-no">2351</span><span id="line-2351"></span>
<span class="source-line-no">2352</span><span id="line-2352"> // TODO test.</span>
<span class="source-line-no">2353</span><span id="line-2353"> continue;</span>
<span class="source-line-no">2354</span><span id="line-2354"> }</span>
<span class="source-line-no">2355</span><span id="line-2355"> if (hbi.getMetaEntry().regionServer == null) {</span>
<span class="source-line-no">2356</span><span id="line-2356"> errors.detail("Skipping region because no region server: " + hbi);</span>
<span class="source-line-no">2357</span><span id="line-2357"> continue;</span>
<span class="source-line-no">2358</span><span id="line-2358"> }</span>
<span class="source-line-no">2359</span><span id="line-2359"> if (hbi.getMetaEntry().getRegionInfo().isOffline()) {</span>
<span class="source-line-no">2360</span><span id="line-2360"> errors.detail("Skipping region because it is offline: " + hbi);</span>
<span class="source-line-no">2361</span><span id="line-2361"> continue;</span>
<span class="source-line-no">2362</span><span id="line-2362"> }</span>
<span class="source-line-no">2363</span><span id="line-2363"> if (hbi.containsOnlyHdfsEdits()) {</span>
<span class="source-line-no">2364</span><span id="line-2364"> errors.detail("Skipping region because it only contains edits" + hbi);</span>
<span class="source-line-no">2365</span><span id="line-2365"> continue;</span>
<span class="source-line-no">2366</span><span id="line-2366"> }</span>
<span class="source-line-no">2367</span><span id="line-2367"></span>
<span class="source-line-no">2368</span><span id="line-2368"> // Missing regionDir or over-deployment is checked elsewhere. Include</span>
<span class="source-line-no">2369</span><span id="line-2369"> // these cases in modTInfo, so we can evaluate those regions as part of</span>
<span class="source-line-no">2370</span><span id="line-2370"> // the region chain in META</span>
<span class="source-line-no">2371</span><span id="line-2371"> // if (hbi.foundRegionDir == null) continue;</span>
<span class="source-line-no">2372</span><span id="line-2372"> // if (hbi.deployedOn.size() != 1) continue;</span>
<span class="source-line-no">2373</span><span id="line-2373"> if (hbi.getDeployedOn().isEmpty()) {</span>
<span class="source-line-no">2374</span><span id="line-2374"> continue;</span>
<span class="source-line-no">2375</span><span id="line-2375"> }</span>
<span class="source-line-no">2376</span><span id="line-2376"></span>
<span class="source-line-no">2377</span><span id="line-2377"> // We should be safe here</span>
<span class="source-line-no">2378</span><span id="line-2378"> TableName tableName = hbi.getMetaEntry().getRegionInfo().getTable();</span>
<span class="source-line-no">2379</span><span id="line-2379"> HbckTableInfo modTInfo = tablesInfo.get(tableName);</span>
<span class="source-line-no">2380</span><span id="line-2380"> if (modTInfo == null) {</span>
<span class="source-line-no">2381</span><span id="line-2381"> modTInfo = new HbckTableInfo(tableName, this);</span>
<span class="source-line-no">2382</span><span id="line-2382"> }</span>
<span class="source-line-no">2383</span><span id="line-2383"> for (ServerName server : hbi.getDeployedOn()) {</span>
<span class="source-line-no">2384</span><span id="line-2384"> modTInfo.addServer(server);</span>
<span class="source-line-no">2385</span><span id="line-2385"> }</span>
<span class="source-line-no">2386</span><span id="line-2386"></span>
<span class="source-line-no">2387</span><span id="line-2387"> if (!hbi.isSkipChecks()) {</span>
<span class="source-line-no">2388</span><span id="line-2388"> modTInfo.addRegionInfo(hbi);</span>
<span class="source-line-no">2389</span><span id="line-2389"> }</span>
<span class="source-line-no">2390</span><span id="line-2390"></span>
<span class="source-line-no">2391</span><span id="line-2391"> tablesInfo.put(tableName, modTInfo);</span>
<span class="source-line-no">2392</span><span id="line-2392"> }</span>
<span class="source-line-no">2393</span><span id="line-2393"></span>
<span class="source-line-no">2394</span><span id="line-2394"> loadTableInfosForTablesWithNoRegion();</span>
<span class="source-line-no">2395</span><span id="line-2395"></span>
<span class="source-line-no">2396</span><span id="line-2396"> logParallelMerge();</span>
<span class="source-line-no">2397</span><span id="line-2397"> for (HbckTableInfo tInfo : tablesInfo.values()) {</span>
<span class="source-line-no">2398</span><span id="line-2398"> TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors);</span>
<span class="source-line-no">2399</span><span id="line-2399"> if (!tInfo.checkRegionChain(handler)) {</span>
<span class="source-line-no">2400</span><span id="line-2400"> errors.report("Found inconsistency in table " + tInfo.getName());</span>
<span class="source-line-no">2401</span><span id="line-2401"> }</span>
<span class="source-line-no">2402</span><span id="line-2402"> }</span>
<span class="source-line-no">2403</span><span id="line-2403"> return tablesInfo;</span>
<span class="source-line-no">2404</span><span id="line-2404"> }</span>
<span class="source-line-no">2405</span><span id="line-2405"></span>
<span class="source-line-no">2406</span><span id="line-2406"> /**</span>
<span class="source-line-no">2407</span><span id="line-2407"> * Loads table info's for tables that may not have been included, since there are no regions</span>
<span class="source-line-no">2408</span><span id="line-2408"> * reported for the table, but table dir is there in hdfs</span>
<span class="source-line-no">2409</span><span id="line-2409"> */</span>
<span class="source-line-no">2410</span><span id="line-2410"> private void loadTableInfosForTablesWithNoRegion() throws IOException {</span>
<span class="source-line-no">2411</span><span id="line-2411"> Map&lt;String, TableDescriptor&gt; allTables = new FSTableDescriptors(getConf()).getAll();</span>
<span class="source-line-no">2412</span><span id="line-2412"> for (TableDescriptor htd : allTables.values()) {</span>
<span class="source-line-no">2413</span><span id="line-2413"> if (checkMetaOnly &amp;&amp; !htd.isMetaTable()) {</span>
<span class="source-line-no">2414</span><span id="line-2414"> continue;</span>
<span class="source-line-no">2415</span><span id="line-2415"> }</span>
<span class="source-line-no">2416</span><span id="line-2416"></span>
<span class="source-line-no">2417</span><span id="line-2417"> TableName tableName = htd.getTableName();</span>
<span class="source-line-no">2418</span><span id="line-2418"> if (isTableIncluded(tableName) &amp;&amp; !tablesInfo.containsKey(tableName)) {</span>
<span class="source-line-no">2419</span><span id="line-2419"> HbckTableInfo tableInfo = new HbckTableInfo(tableName, this);</span>
<span class="source-line-no">2420</span><span id="line-2420"> tableInfo.htds.add(htd);</span>
<span class="source-line-no">2421</span><span id="line-2421"> tablesInfo.put(htd.getTableName(), tableInfo);</span>
<span class="source-line-no">2422</span><span id="line-2422"> }</span>
<span class="source-line-no">2423</span><span id="line-2423"> }</span>
<span class="source-line-no">2424</span><span id="line-2424"> }</span>
<span class="source-line-no">2425</span><span id="line-2425"></span>
<span class="source-line-no">2426</span><span id="line-2426"> /**</span>
<span class="source-line-no">2427</span><span id="line-2427"> * Merge hdfs data by moving from contained HbckRegionInfo into targetRegionDir.</span>
<span class="source-line-no">2428</span><span id="line-2428"> * @return number of file move fixes done to merge regions.</span>
<span class="source-line-no">2429</span><span id="line-2429"> */</span>
<span class="source-line-no">2430</span><span id="line-2430"> public int mergeRegionDirs(Path targetRegionDir, HbckRegionInfo contained) throws IOException {</span>
<span class="source-line-no">2431</span><span id="line-2431"> int fileMoves = 0;</span>
<span class="source-line-no">2432</span><span id="line-2432"> String thread = Thread.currentThread().getName();</span>
<span class="source-line-no">2433</span><span id="line-2433"> LOG.debug("[" + thread + "] Contained region dir after close and pause");</span>
<span class="source-line-no">2434</span><span id="line-2434"> debugLsr(contained.getHdfsRegionDir());</span>
<span class="source-line-no">2435</span><span id="line-2435"></span>
<span class="source-line-no">2436</span><span id="line-2436"> // rename the contained into the container.</span>
<span class="source-line-no">2437</span><span id="line-2437"> FileSystem fs = targetRegionDir.getFileSystem(getConf());</span>
<span class="source-line-no">2438</span><span id="line-2438"> FileStatus[] dirs = null;</span>
<span class="source-line-no">2439</span><span id="line-2439"> try {</span>
<span class="source-line-no">2440</span><span id="line-2440"> dirs = fs.listStatus(contained.getHdfsRegionDir());</span>
<span class="source-line-no">2441</span><span id="line-2441"> } catch (FileNotFoundException fnfe) {</span>
<span class="source-line-no">2442</span><span id="line-2442"> // region we are attempting to merge in is not present! Since this is a merge, there is</span>
<span class="source-line-no">2443</span><span id="line-2443"> // no harm skipping this region if it does not exist.</span>
<span class="source-line-no">2444</span><span id="line-2444"> if (!fs.exists(contained.getHdfsRegionDir())) {</span>
<span class="source-line-no">2445</span><span id="line-2445"> LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()</span>
<span class="source-line-no">2446</span><span id="line-2446"> + " is missing. Assuming already sidelined or moved.");</span>
<span class="source-line-no">2447</span><span id="line-2447"> } else {</span>
<span class="source-line-no">2448</span><span id="line-2448"> sidelineRegionDir(fs, contained);</span>
<span class="source-line-no">2449</span><span id="line-2449"> }</span>
<span class="source-line-no">2450</span><span id="line-2450"> return fileMoves;</span>
<span class="source-line-no">2451</span><span id="line-2451"> }</span>
<span class="source-line-no">2452</span><span id="line-2452"></span>
<span class="source-line-no">2453</span><span id="line-2453"> if (dirs == null) {</span>
<span class="source-line-no">2454</span><span id="line-2454"> if (!fs.exists(contained.getHdfsRegionDir())) {</span>
<span class="source-line-no">2455</span><span id="line-2455"> LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir()</span>
<span class="source-line-no">2456</span><span id="line-2456"> + " already sidelined.");</span>
<span class="source-line-no">2457</span><span id="line-2457"> } else {</span>
<span class="source-line-no">2458</span><span id="line-2458"> sidelineRegionDir(fs, contained);</span>
<span class="source-line-no">2459</span><span id="line-2459"> }</span>
<span class="source-line-no">2460</span><span id="line-2460"> return fileMoves;</span>
<span class="source-line-no">2461</span><span id="line-2461"> }</span>
<span class="source-line-no">2462</span><span id="line-2462"></span>
<span class="source-line-no">2463</span><span id="line-2463"> for (FileStatus cf : dirs) {</span>
<span class="source-line-no">2464</span><span id="line-2464"> Path src = cf.getPath();</span>
<span class="source-line-no">2465</span><span id="line-2465"> Path dst = new Path(targetRegionDir, src.getName());</span>
<span class="source-line-no">2466</span><span id="line-2466"></span>
<span class="source-line-no">2467</span><span id="line-2467"> if (src.getName().equals(HRegionFileSystem.REGION_INFO_FILE)) {</span>
<span class="source-line-no">2468</span><span id="line-2468"> // do not copy the old .regioninfo file.</span>
<span class="source-line-no">2469</span><span id="line-2469"> continue;</span>
<span class="source-line-no">2470</span><span id="line-2470"> }</span>
<span class="source-line-no">2471</span><span id="line-2471"></span>
<span class="source-line-no">2472</span><span id="line-2472"> if (src.getName().equals(HConstants.HREGION_OLDLOGDIR_NAME)) {</span>
<span class="source-line-no">2473</span><span id="line-2473"> // do not copy the .oldlogs files</span>
<span class="source-line-no">2474</span><span id="line-2474"> continue;</span>
<span class="source-line-no">2475</span><span id="line-2475"> }</span>
<span class="source-line-no">2476</span><span id="line-2476"></span>
<span class="source-line-no">2477</span><span id="line-2477"> LOG.info("[" + thread + "] Moving files from " + src + " into containing region " + dst);</span>
<span class="source-line-no">2478</span><span id="line-2478"> // FileSystem.rename is inconsistent with directories -- if the</span>
<span class="source-line-no">2479</span><span id="line-2479"> // dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,</span>
<span class="source-line-no">2480</span><span id="line-2480"> // it moves the src into the dst dir resulting in (foo/a/b). If</span>
<span class="source-line-no">2481</span><span id="line-2481"> // the dst does not exist, and the src a dir, src becomes dst. (foo/b)</span>
<span class="source-line-no">2482</span><span id="line-2482"> for (FileStatus hfile : fs.listStatus(src)) {</span>
<span class="source-line-no">2483</span><span id="line-2483"> boolean success = fs.rename(hfile.getPath(), dst);</span>
<span class="source-line-no">2484</span><span id="line-2484"> if (success) {</span>
<span class="source-line-no">2485</span><span id="line-2485"> fileMoves++;</span>
<span class="source-line-no">2486</span><span id="line-2486"> }</span>
<span class="source-line-no">2487</span><span id="line-2487"> }</span>
<span class="source-line-no">2488</span><span id="line-2488"> LOG.debug("[" + thread + "] Sideline directory contents:");</span>
<span class="source-line-no">2489</span><span id="line-2489"> debugLsr(targetRegionDir);</span>
<span class="source-line-no">2490</span><span id="line-2490"> }</span>
<span class="source-line-no">2491</span><span id="line-2491"></span>
<span class="source-line-no">2492</span><span id="line-2492"> // if all success.</span>
<span class="source-line-no">2493</span><span id="line-2493"> sidelineRegionDir(fs, contained);</span>
<span class="source-line-no">2494</span><span id="line-2494"> LOG.info("[" + thread + "] Sidelined region dir " + contained.getHdfsRegionDir() + " into "</span>
<span class="source-line-no">2495</span><span id="line-2495"> + getSidelineDir());</span>
<span class="source-line-no">2496</span><span id="line-2496"> debugLsr(contained.getHdfsRegionDir());</span>
<span class="source-line-no">2497</span><span id="line-2497"></span>
<span class="source-line-no">2498</span><span id="line-2498"> return fileMoves;</span>
<span class="source-line-no">2499</span><span id="line-2499"> }</span>
<span class="source-line-no">2500</span><span id="line-2500"></span>
<span class="source-line-no">2501</span><span id="line-2501"> static class WorkItemOverlapMerge implements Callable&lt;Void&gt; {</span>
<span class="source-line-no">2502</span><span id="line-2502"> private TableIntegrityErrorHandler handler;</span>
<span class="source-line-no">2503</span><span id="line-2503"> Collection&lt;HbckRegionInfo&gt; overlapgroup;</span>
<span class="source-line-no">2504</span><span id="line-2504"></span>
<span class="source-line-no">2505</span><span id="line-2505"> WorkItemOverlapMerge(Collection&lt;HbckRegionInfo&gt; overlapgroup,</span>
<span class="source-line-no">2506</span><span id="line-2506"> TableIntegrityErrorHandler handler) {</span>
<span class="source-line-no">2507</span><span id="line-2507"> this.handler = handler;</span>
<span class="source-line-no">2508</span><span id="line-2508"> this.overlapgroup = overlapgroup;</span>
<span class="source-line-no">2509</span><span id="line-2509"> }</span>
<span class="source-line-no">2510</span><span id="line-2510"></span>
<span class="source-line-no">2511</span><span id="line-2511"> @Override</span>
<span class="source-line-no">2512</span><span id="line-2512"> public Void call() throws Exception {</span>
<span class="source-line-no">2513</span><span id="line-2513"> handler.handleOverlapGroup(overlapgroup);</span>
<span class="source-line-no">2514</span><span id="line-2514"> return null;</span>
<span class="source-line-no">2515</span><span id="line-2515"> }</span>
<span class="source-line-no">2516</span><span id="line-2516"> }</span>
<span class="source-line-no">2517</span><span id="line-2517"></span>
<span class="source-line-no">2518</span><span id="line-2518"> /**</span>
<span class="source-line-no">2519</span><span id="line-2519"> * Return a list of user-space table names whose metadata have not been modified in the last few</span>
<span class="source-line-no">2520</span><span id="line-2520"> * milliseconds specified by timelag if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER,</span>
<span class="source-line-no">2521</span><span id="line-2521"> * STARTCODE_QUALIFIER, SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last</span>
<span class="source-line-no">2522</span><span id="line-2522"> * milliseconds specified by timelag, then the table is a candidate to be returned.</span>
<span class="source-line-no">2523</span><span id="line-2523"> * @return tables that have not been modified recently</span>
<span class="source-line-no">2524</span><span id="line-2524"> * @throws IOException if an error is encountered</span>
<span class="source-line-no">2525</span><span id="line-2525"> */</span>
<span class="source-line-no">2526</span><span id="line-2526"> TableDescriptor[] getTables(AtomicInteger numSkipped) {</span>
<span class="source-line-no">2527</span><span id="line-2527"> List&lt;TableName&gt; tableNames = new ArrayList&lt;&gt;();</span>
<span class="source-line-no">2528</span><span id="line-2528"> long now = EnvironmentEdgeManager.currentTime();</span>
<span class="source-line-no">2529</span><span id="line-2529"></span>
<span class="source-line-no">2530</span><span id="line-2530"> for (HbckRegionInfo hbi : regionInfoMap.values()) {</span>
<span class="source-line-no">2531</span><span id="line-2531"> HbckRegionInfo.MetaEntry info = hbi.getMetaEntry();</span>
<span class="source-line-no">2532</span><span id="line-2532"></span>
<span class="source-line-no">2533</span><span id="line-2533"> // if the start key is zero, then we have found the first region of a table.</span>
<span class="source-line-no">2534</span><span id="line-2534"> // pick only those tables that were not modified in the last few milliseconds.</span>
<span class="source-line-no">2535</span><span id="line-2535"> if (</span>
<span class="source-line-no">2536</span><span id="line-2536"> info != null &amp;&amp; info.getRegionInfo().getStartKey().length == 0</span>
<span class="source-line-no">2537</span><span id="line-2537"> &amp;&amp; !info.getRegionInfo().isMetaRegion()</span>
<span class="source-line-no">2538</span><span id="line-2538"> ) {</span>
<span class="source-line-no">2539</span><span id="line-2539"> if (info.modTime + timelag &lt; now) {</span>
<span class="source-line-no">2540</span><span id="line-2540"> tableNames.add(info.getRegionInfo().getTable());</span>
<span class="source-line-no">2541</span><span id="line-2541"> } else {</span>
<span class="source-line-no">2542</span><span id="line-2542"> numSkipped.incrementAndGet(); // one more in-flux table</span>
<span class="source-line-no">2543</span><span id="line-2543"> }</span>
<span class="source-line-no">2544</span><span id="line-2544"> }</span>
<span class="source-line-no">2545</span><span id="line-2545"> }</span>
<span class="source-line-no">2546</span><span id="line-2546"> return getTableDescriptors(tableNames);</span>
<span class="source-line-no">2547</span><span id="line-2547"> }</span>
<span class="source-line-no">2548</span><span id="line-2548"></span>
<span class="source-line-no">2549</span><span id="line-2549"> TableDescriptor[] getTableDescriptors(List&lt;TableName&gt; tableNames) {</span>
<span class="source-line-no">2550</span><span id="line-2550"> LOG.info("getTableDescriptors == tableNames =&gt; " + tableNames);</span>
<span class="source-line-no">2551</span><span id="line-2551"> try (Connection conn = ConnectionFactory.createConnection(getConf());</span>
<span class="source-line-no">2552</span><span id="line-2552"> Admin admin = conn.getAdmin()) {</span>
<span class="source-line-no">2553</span><span id="line-2553"> List&lt;TableDescriptor&gt; tds = admin.listTableDescriptors(tableNames);</span>
<span class="source-line-no">2554</span><span id="line-2554"> return tds.toArray(new TableDescriptor[tds.size()]);</span>
<span class="source-line-no">2555</span><span id="line-2555"> } catch (IOException e) {</span>
<span class="source-line-no">2556</span><span id="line-2556"> LOG.debug("Exception getting table descriptors", e);</span>
<span class="source-line-no">2557</span><span id="line-2557"> }</span>
<span class="source-line-no">2558</span><span id="line-2558"> return new TableDescriptor[0];</span>
<span class="source-line-no">2559</span><span id="line-2559"> }</span>
<span class="source-line-no">2560</span><span id="line-2560"></span>
<span class="source-line-no">2561</span><span id="line-2561"> /**</span>
<span class="source-line-no">2562</span><span id="line-2562"> * Gets the entry in regionInfo corresponding to the the given encoded region name. If the region</span>
<span class="source-line-no">2563</span><span id="line-2563"> * has not been seen yet, a new entry is added and returned.</span>
<span class="source-line-no">2564</span><span id="line-2564"> */</span>
<span class="source-line-no">2565</span><span id="line-2565"> private synchronized HbckRegionInfo getOrCreateInfo(String name) {</span>
<span class="source-line-no">2566</span><span id="line-2566"> HbckRegionInfo hbi = regionInfoMap.get(name);</span>
<span class="source-line-no">2567</span><span id="line-2567"> if (hbi == null) {</span>
<span class="source-line-no">2568</span><span id="line-2568"> hbi = new HbckRegionInfo(null);</span>
<span class="source-line-no">2569</span><span id="line-2569"> regionInfoMap.put(name, hbi);</span>
<span class="source-line-no">2570</span><span id="line-2570"> }</span>
<span class="source-line-no">2571</span><span id="line-2571"> return hbi;</span>
<span class="source-line-no">2572</span><span id="line-2572"> }</span>
<span class="source-line-no">2573</span><span id="line-2573"></span>
<span class="source-line-no">2574</span><span id="line-2574"> private void checkAndFixReplication() throws ReplicationException, IOException {</span>
<span class="source-line-no">2575</span><span id="line-2575"> ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);</span>
<span class="source-line-no">2576</span><span id="line-2576"></span>
<span class="source-line-no">2577</span><span id="line-2577"> if (!checker.checkHasDataInQueues()) {</span>
<span class="source-line-no">2578</span><span id="line-2578"> return;</span>
<span class="source-line-no">2579</span><span id="line-2579"> }</span>
<span class="source-line-no">2580</span><span id="line-2580"></span>
<span class="source-line-no">2581</span><span id="line-2581"> checker.checkUnDeletedQueues();</span>
<span class="source-line-no">2582</span><span id="line-2582"></span>
<span class="source-line-no">2583</span><span id="line-2583"> if (checker.hasUnDeletedQueues() &amp;&amp; this.fixReplication) {</span>
<span class="source-line-no">2584</span><span id="line-2584"> checker.fixUnDeletedQueues();</span>
<span class="source-line-no">2585</span><span id="line-2585"> setShouldRerun();</span>
<span class="source-line-no">2586</span><span id="line-2586"> }</span>
<span class="source-line-no">2587</span><span id="line-2587"> }</span>
<span class="source-line-no">2588</span><span id="line-2588"></span>
<span class="source-line-no">2589</span><span id="line-2589"> /**</span>
<span class="source-line-no">2590</span><span id="line-2590"> * Check values in regionInfo for hbase:meta Check if zero or more than one regions with</span>
<span class="source-line-no">2591</span><span id="line-2591"> * hbase:meta are found. If there are inconsistencies (i.e. zero or more than one regions pretend</span>
<span class="source-line-no">2592</span><span id="line-2592"> * to be holding the hbase:meta) try to fix that and report an error.</span>
<span class="source-line-no">2593</span><span id="line-2593"> * @throws IOException from HBaseFsckRepair functions</span>
<span class="source-line-no">2594</span><span id="line-2594"> */</span>
<span class="source-line-no">2595</span><span id="line-2595"> boolean checkMetaRegion() throws IOException, KeeperException, InterruptedException {</span>
<span class="source-line-no">2596</span><span id="line-2596"> Map&lt;Integer, HbckRegionInfo&gt; metaRegions = new HashMap&lt;&gt;();</span>
<span class="source-line-no">2597</span><span id="line-2597"> for (HbckRegionInfo value : regionInfoMap.values()) {</span>
<span class="source-line-no">2598</span><span id="line-2598"> if (value.getMetaEntry() != null &amp;&amp; value.getMetaEntry().getRegionInfo().isMetaRegion()) {</span>
<span class="source-line-no">2599</span><span id="line-2599"> metaRegions.put(value.getReplicaId(), value);</span>
<span class="source-line-no">2600</span><span id="line-2600"> }</span>
<span class="source-line-no">2601</span><span id="line-2601"> }</span>
<span class="source-line-no">2602</span><span id="line-2602"> int metaReplication = admin.getDescriptor(TableName.META_TABLE_NAME).getRegionReplication();</span>
<span class="source-line-no">2603</span><span id="line-2603"> boolean noProblem = true;</span>
<span class="source-line-no">2604</span><span id="line-2604"> // There will be always entries in regionInfoMap corresponding to hbase:meta &amp; its replicas</span>
<span class="source-line-no">2605</span><span id="line-2605"> // Check the deployed servers. It should be exactly one server for each replica.</span>
<span class="source-line-no">2606</span><span id="line-2606"> for (int i = 0; i &lt; metaReplication; i++) {</span>
<span class="source-line-no">2607</span><span id="line-2607"> HbckRegionInfo metaHbckRegionInfo = metaRegions.remove(i);</span>
<span class="source-line-no">2608</span><span id="line-2608"> List&lt;ServerName&gt; servers = new ArrayList&lt;&gt;();</span>
<span class="source-line-no">2609</span><span id="line-2609"> if (metaHbckRegionInfo != null) {</span>
<span class="source-line-no">2610</span><span id="line-2610"> servers = metaHbckRegionInfo.getDeployedOn();</span>
<span class="source-line-no">2611</span><span id="line-2611"> }</span>
<span class="source-line-no">2612</span><span id="line-2612"> if (servers.size() != 1) {</span>
<span class="source-line-no">2613</span><span id="line-2613"> noProblem = false;</span>
<span class="source-line-no">2614</span><span id="line-2614"> if (servers.isEmpty()) {</span>
<span class="source-line-no">2615</span><span id="line-2615"> assignMetaReplica(i);</span>
<span class="source-line-no">2616</span><span id="line-2616"> } else if (servers.size() &gt; 1) {</span>
<span class="source-line-no">2617</span><span id="line-2617"> errors.reportError(ERROR_CODE.MULTI_META_REGION, "hbase:meta, replicaId "</span>
<span class="source-line-no">2618</span><span id="line-2618"> + metaHbckRegionInfo.getReplicaId() + " is found on more than one region.");</span>
<span class="source-line-no">2619</span><span id="line-2619"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2620</span><span id="line-2620"> errors.print("Trying to fix a problem with hbase:meta, replicaId "</span>
<span class="source-line-no">2621</span><span id="line-2621"> + metaHbckRegionInfo.getReplicaId() + "..");</span>
<span class="source-line-no">2622</span><span id="line-2622"> setShouldRerun();</span>
<span class="source-line-no">2623</span><span id="line-2623"> // try fix it (treat is a dupe assignment)</span>
<span class="source-line-no">2624</span><span id="line-2624"> HBaseFsckRepair.fixMultiAssignment(connection,</span>
<span class="source-line-no">2625</span><span id="line-2625"> metaHbckRegionInfo.getMetaEntry().getRegionInfo(), servers);</span>
<span class="source-line-no">2626</span><span id="line-2626"> }</span>
<span class="source-line-no">2627</span><span id="line-2627"> }</span>
<span class="source-line-no">2628</span><span id="line-2628"> }</span>
<span class="source-line-no">2629</span><span id="line-2629"> }</span>
<span class="source-line-no">2630</span><span id="line-2630"> // unassign whatever is remaining in metaRegions. They are excess replicas.</span>
<span class="source-line-no">2631</span><span id="line-2631"> for (Map.Entry&lt;Integer, HbckRegionInfo&gt; entry : metaRegions.entrySet()) {</span>
<span class="source-line-no">2632</span><span id="line-2632"> noProblem = false;</span>
<span class="source-line-no">2633</span><span id="line-2633"> errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED,</span>
<span class="source-line-no">2634</span><span id="line-2634"> "hbase:meta replicas are deployed in excess. Configured " + metaReplication + ", deployed "</span>
<span class="source-line-no">2635</span><span id="line-2635"> + metaRegions.size());</span>
<span class="source-line-no">2636</span><span id="line-2636"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2637</span><span id="line-2637"> errors.print(</span>
<span class="source-line-no">2638</span><span id="line-2638"> "Trying to undeploy excess replica, replicaId: " + entry.getKey() + " of hbase:meta..");</span>
<span class="source-line-no">2639</span><span id="line-2639"> setShouldRerun();</span>
<span class="source-line-no">2640</span><span id="line-2640"> unassignMetaReplica(entry.getValue());</span>
<span class="source-line-no">2641</span><span id="line-2641"> }</span>
<span class="source-line-no">2642</span><span id="line-2642"> }</span>
<span class="source-line-no">2643</span><span id="line-2643"> // if noProblem is false, rerun hbck with hopefully fixed META</span>
<span class="source-line-no">2644</span><span id="line-2644"> // if noProblem is true, no errors, so continue normally</span>
<span class="source-line-no">2645</span><span id="line-2645"> return noProblem;</span>
<span class="source-line-no">2646</span><span id="line-2646"> }</span>
<span class="source-line-no">2647</span><span id="line-2647"></span>
<span class="source-line-no">2648</span><span id="line-2648"> private void unassignMetaReplica(HbckRegionInfo hi)</span>
<span class="source-line-no">2649</span><span id="line-2649"> throws IOException, InterruptedException, KeeperException {</span>
<span class="source-line-no">2650</span><span id="line-2650"> undeployRegions(hi);</span>
<span class="source-line-no">2651</span><span id="line-2651"> ZKUtil.deleteNode(zkw,</span>
<span class="source-line-no">2652</span><span id="line-2652"> zkw.getZNodePaths().getZNodeForReplica(hi.getMetaEntry().getRegionInfo().getReplicaId()));</span>
<span class="source-line-no">2653</span><span id="line-2653"> }</span>
<span class="source-line-no">2654</span><span id="line-2654"></span>
<span class="source-line-no">2655</span><span id="line-2655"> private void assignMetaReplica(int replicaId)</span>
<span class="source-line-no">2656</span><span id="line-2656"> throws IOException, KeeperException, InterruptedException {</span>
<span class="source-line-no">2657</span><span id="line-2657"> errors.reportError(ERROR_CODE.NO_META_REGION,</span>
<span class="source-line-no">2658</span><span id="line-2658"> "hbase:meta, replicaId " + replicaId + " is not found on any region.");</span>
<span class="source-line-no">2659</span><span id="line-2659"> if (shouldFixAssignments()) {</span>
<span class="source-line-no">2660</span><span id="line-2660"> errors.print("Trying to fix a problem with hbase:meta..");</span>
<span class="source-line-no">2661</span><span id="line-2661"> setShouldRerun();</span>
<span class="source-line-no">2662</span><span id="line-2662"> // try to fix it (treat it as unassigned region)</span>
<span class="source-line-no">2663</span><span id="line-2663"> RegionInfo h = RegionReplicaUtil</span>
<span class="source-line-no">2664</span><span id="line-2664"> .getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);</span>
<span class="source-line-no">2665</span><span id="line-2665"> HBaseFsckRepair.fixUnassigned(admin, h);</span>
<span class="source-line-no">2666</span><span id="line-2666"> HBaseFsckRepair.waitUntilAssigned(admin, h);</span>
<span class="source-line-no">2667</span><span id="line-2667"> }</span>
<span class="source-line-no">2668</span><span id="line-2668"> }</span>
<span class="source-line-no">2669</span><span id="line-2669"></span>
<span class="source-line-no">2670</span><span id="line-2670"> /**</span>
<span class="source-line-no">2671</span><span id="line-2671"> * Scan hbase:meta, adding all regions found to the regionInfo map.</span>
<span class="source-line-no">2672</span><span id="line-2672"> * @throws IOException if an error is encountered</span>
<span class="source-line-no">2673</span><span id="line-2673"> */</span>
<span class="source-line-no">2674</span><span id="line-2674"> boolean loadMetaEntries() throws IOException {</span>
<span class="source-line-no">2675</span><span id="line-2675"> ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {</span>
<span class="source-line-no">2676</span><span id="line-2676"> int countRecord = 1;</span>
<span class="source-line-no">2677</span><span id="line-2677"></span>
<span class="source-line-no">2678</span><span id="line-2678"> // comparator to sort KeyValues with latest modtime</span>
<span class="source-line-no">2679</span><span id="line-2679"> final Comparator&lt;Cell&gt; comp = new Comparator&lt;Cell&gt;() {</span>
<span class="source-line-no">2680</span><span id="line-2680"> @Override</span>
<span class="source-line-no">2681</span><span id="line-2681"> public int compare(Cell k1, Cell k2) {</span>
<span class="source-line-no">2682</span><span id="line-2682"> return Long.compare(k1.getTimestamp(), k2.getTimestamp());</span>
<span class="source-line-no">2683</span><span id="line-2683"> }</span>
<span class="source-line-no">2684</span><span id="line-2684"> };</span>
<span class="source-line-no">2685</span><span id="line-2685"></span>
<span class="source-line-no">2686</span><span id="line-2686"> @Override</span>
<span class="source-line-no">2687</span><span id="line-2687"> public boolean visit(Result result) throws IOException {</span>
<span class="source-line-no">2688</span><span id="line-2688"> try {</span>
<span class="source-line-no">2689</span><span id="line-2689"></span>
<span class="source-line-no">2690</span><span id="line-2690"> // record the latest modification of this META record</span>
<span class="source-line-no">2691</span><span id="line-2691"> long ts = Collections.max(result.listCells(), comp).getTimestamp();</span>
<span class="source-line-no">2692</span><span id="line-2692"> RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result);</span>
<span class="source-line-no">2693</span><span id="line-2693"> if (rl == null) {</span>
<span class="source-line-no">2694</span><span id="line-2694"> emptyRegionInfoQualifiers.add(result);</span>
<span class="source-line-no">2695</span><span id="line-2695"> errors.reportError(ERROR_CODE.EMPTY_META_CELL,</span>
<span class="source-line-no">2696</span><span id="line-2696"> "Empty REGIONINFO_QUALIFIER found in hbase:meta");</span>
<span class="source-line-no">2697</span><span id="line-2697"> return true;</span>
<span class="source-line-no">2698</span><span id="line-2698"> }</span>
<span class="source-line-no">2699</span><span id="line-2699"> ServerName sn = null;</span>
<span class="source-line-no">2700</span><span id="line-2700"> if (</span>
<span class="source-line-no">2701</span><span id="line-2701"> rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID) == null</span>
<span class="source-line-no">2702</span><span id="line-2702"> || rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegion() == null</span>
<span class="source-line-no">2703</span><span id="line-2703"> ) {</span>
<span class="source-line-no">2704</span><span id="line-2704"> emptyRegionInfoQualifiers.add(result);</span>
<span class="source-line-no">2705</span><span id="line-2705"> errors.reportError(ERROR_CODE.EMPTY_META_CELL,</span>
<span class="source-line-no">2706</span><span id="line-2706"> "Empty REGIONINFO_QUALIFIER found in hbase:meta");</span>
<span class="source-line-no">2707</span><span id="line-2707"> return true;</span>
<span class="source-line-no">2708</span><span id="line-2708"> }</span>
<span class="source-line-no">2709</span><span id="line-2709"> RegionInfo hri = rl.getRegionLocation(RegionInfo.DEFAULT_REPLICA_ID).getRegion();</span>
<span class="source-line-no">2710</span><span id="line-2710"> if (!(isTableIncluded(hri.getTable()) || hri.isMetaRegion())) {</span>
<span class="source-line-no">2711</span><span id="line-2711"> return true;</span>
<span class="source-line-no">2712</span><span id="line-2712"> }</span>
<span class="source-line-no">2713</span><span id="line-2713"> PairOfSameType&lt;RegionInfo&gt; daughters = MetaTableAccessor.getDaughterRegions(result);</span>
<span class="source-line-no">2714</span><span id="line-2714"> for (HRegionLocation h : rl.getRegionLocations()) {</span>
<span class="source-line-no">2715</span><span id="line-2715"> if (h == null || h.getRegion() == null) {</span>
<span class="source-line-no">2716</span><span id="line-2716"> continue;</span>
<span class="source-line-no">2717</span><span id="line-2717"> }</span>
<span class="source-line-no">2718</span><span id="line-2718"> sn = h.getServerName();</span>
<span class="source-line-no">2719</span><span id="line-2719"> hri = h.getRegion();</span>
<span class="source-line-no">2720</span><span id="line-2720"></span>
<span class="source-line-no">2721</span><span id="line-2721"> HbckRegionInfo.MetaEntry m = null;</span>
<span class="source-line-no">2722</span><span id="line-2722"> if (hri.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {</span>
<span class="source-line-no">2723</span><span id="line-2723"> m = new HbckRegionInfo.MetaEntry(hri, sn, ts, daughters.getFirst(),</span>
<span class="source-line-no">2724</span><span id="line-2724"> daughters.getSecond());</span>
<span class="source-line-no">2725</span><span id="line-2725"> } else {</span>
<span class="source-line-no">2726</span><span id="line-2726"> m = new HbckRegionInfo.MetaEntry(hri, sn, ts, null, null);</span>
<span class="source-line-no">2727</span><span id="line-2727"> }</span>
<span class="source-line-no">2728</span><span id="line-2728"> HbckRegionInfo previous = regionInfoMap.get(hri.getEncodedName());</span>
<span class="source-line-no">2729</span><span id="line-2729"> if (previous == null) {</span>
<span class="source-line-no">2730</span><span id="line-2730"> regionInfoMap.put(hri.getEncodedName(), new HbckRegionInfo(m));</span>
<span class="source-line-no">2731</span><span id="line-2731"> } else if (previous.getMetaEntry() == null) {</span>
<span class="source-line-no">2732</span><span id="line-2732"> previous.setMetaEntry(m);</span>
<span class="source-line-no">2733</span><span id="line-2733"> } else {</span>
<span class="source-line-no">2734</span><span id="line-2734"> throw new IOException("Two entries in hbase:meta are same " + previous);</span>
<span class="source-line-no">2735</span><span id="line-2735"> }</span>
<span class="source-line-no">2736</span><span id="line-2736"> }</span>
<span class="source-line-no">2737</span><span id="line-2737"> List&lt;RegionInfo&gt; mergeParents = CatalogFamilyFormat.getMergeRegions(result.rawCells());</span>
<span class="source-line-no">2738</span><span id="line-2738"> if (mergeParents != null) {</span>
<span class="source-line-no">2739</span><span id="line-2739"> for (RegionInfo mergeRegion : mergeParents) {</span>
<span class="source-line-no">2740</span><span id="line-2740"> if (mergeRegion != null) {</span>
<span class="source-line-no">2741</span><span id="line-2741"> // This region is already being merged</span>
<span class="source-line-no">2742</span><span id="line-2742"> HbckRegionInfo hbInfo = getOrCreateInfo(mergeRegion.getEncodedName());</span>
<span class="source-line-no">2743</span><span id="line-2743"> hbInfo.setMerged(true);</span>
<span class="source-line-no">2744</span><span id="line-2744"> }</span>
<span class="source-line-no">2745</span><span id="line-2745"> }</span>
<span class="source-line-no">2746</span><span id="line-2746"> }</span>
<span class="source-line-no">2747</span><span id="line-2747"></span>
<span class="source-line-no">2748</span><span id="line-2748"> // show proof of progress to the user, once for every 100 records.</span>
<span class="source-line-no">2749</span><span id="line-2749"> if (countRecord % 100 == 0) {</span>
<span class="source-line-no">2750</span><span id="line-2750"> errors.progress();</span>
<span class="source-line-no">2751</span><span id="line-2751"> }</span>
<span class="source-line-no">2752</span><span id="line-2752"> countRecord++;</span>
<span class="source-line-no">2753</span><span id="line-2753"> return true;</span>
<span class="source-line-no">2754</span><span id="line-2754"> } catch (RuntimeException e) {</span>
<span class="source-line-no">2755</span><span id="line-2755"> LOG.error("Result=" + result);</span>
<span class="source-line-no">2756</span><span id="line-2756"> throw e;</span>
<span class="source-line-no">2757</span><span id="line-2757"> }</span>
<span class="source-line-no">2758</span><span id="line-2758"> }</span>
<span class="source-line-no">2759</span><span id="line-2759"> };</span>
<span class="source-line-no">2760</span><span id="line-2760"> if (!checkMetaOnly) {</span>
<span class="source-line-no">2761</span><span id="line-2761"> // Scan hbase:meta to pick up user regions</span>
<span class="source-line-no">2762</span><span id="line-2762"> MetaTableAccessor.fullScanRegions(connection, visitor);</span>
<span class="source-line-no">2763</span><span id="line-2763"> }</span>
<span class="source-line-no">2764</span><span id="line-2764"></span>
<span class="source-line-no">2765</span><span id="line-2765"> errors.print("");</span>
<span class="source-line-no">2766</span><span id="line-2766"> return true;</span>
<span class="source-line-no">2767</span><span id="line-2767"> }</span>
<span class="source-line-no">2768</span><span id="line-2768"></span>
<span class="source-line-no">2769</span><span id="line-2769"> /**</span>
<span class="source-line-no">2770</span><span id="line-2770"> * Prints summary of all tables found on the system.</span>
<span class="source-line-no">2771</span><span id="line-2771"> */</span>
<span class="source-line-no">2772</span><span id="line-2772"> private void printTableSummary(SortedMap&lt;TableName, HbckTableInfo&gt; tablesInfo) {</span>
<span class="source-line-no">2773</span><span id="line-2773"> StringBuilder sb = new StringBuilder();</span>
<span class="source-line-no">2774</span><span id="line-2774"> int numOfSkippedRegions;</span>
<span class="source-line-no">2775</span><span id="line-2775"> errors.print("Summary:");</span>
<span class="source-line-no">2776</span><span id="line-2776"> for (HbckTableInfo tInfo : tablesInfo.values()) {</span>
<span class="source-line-no">2777</span><span id="line-2777"> numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName()))</span>
<span class="source-line-no">2778</span><span id="line-2778"> ? skippedRegions.get(tInfo.getName()).size()</span>
<span class="source-line-no">2779</span><span id="line-2779"> : 0;</span>
<span class="source-line-no">2780</span><span id="line-2780"></span>
<span class="source-line-no">2781</span><span id="line-2781"> if (errors.tableHasErrors(tInfo)) {</span>
<span class="source-line-no">2782</span><span id="line-2782"> errors.print("Table " + tInfo.getName() + " is inconsistent.");</span>
<span class="source-line-no">2783</span><span id="line-2783"> } else if (numOfSkippedRegions &gt; 0) {</span>
<span class="source-line-no">2784</span><span id="line-2784"> errors.print("Table " + tInfo.getName() + " is okay (with " + numOfSkippedRegions</span>
<span class="source-line-no">2785</span><span id="line-2785"> + " skipped regions).");</span>
<span class="source-line-no">2786</span><span id="line-2786"> } else {</span>
<span class="source-line-no">2787</span><span id="line-2787"> errors.print("Table " + tInfo.getName() + " is okay.");</span>
<span class="source-line-no">2788</span><span id="line-2788"> }</span>
<span class="source-line-no">2789</span><span id="line-2789"> errors.print(" Number of regions: " + tInfo.getNumRegions());</span>
<span class="source-line-no">2790</span><span id="line-2790"> if (numOfSkippedRegions &gt; 0) {</span>
<span class="source-line-no">2791</span><span id="line-2791"> Set&lt;String&gt; skippedRegionStrings = skippedRegions.get(tInfo.getName());</span>
<span class="source-line-no">2792</span><span id="line-2792"> System.out.println(" Number of skipped regions: " + numOfSkippedRegions);</span>
<span class="source-line-no">2793</span><span id="line-2793"> System.out.println(" List of skipped regions:");</span>
<span class="source-line-no">2794</span><span id="line-2794"> for (String sr : skippedRegionStrings) {</span>
<span class="source-line-no">2795</span><span id="line-2795"> System.out.println(" " + sr);</span>
<span class="source-line-no">2796</span><span id="line-2796"> }</span>
<span class="source-line-no">2797</span><span id="line-2797"> }</span>
<span class="source-line-no">2798</span><span id="line-2798"> sb.setLength(0); // clear out existing buffer, if any.</span>
<span class="source-line-no">2799</span><span id="line-2799"> sb.append(" Deployed on: ");</span>
<span class="source-line-no">2800</span><span id="line-2800"> for (ServerName server : tInfo.deployedOn) {</span>
<span class="source-line-no">2801</span><span id="line-2801"> sb.append(" " + server.toString());</span>
<span class="source-line-no">2802</span><span id="line-2802"> }</span>
<span class="source-line-no">2803</span><span id="line-2803"> errors.print(sb.toString());</span>
<span class="source-line-no">2804</span><span id="line-2804"> }</span>
<span class="source-line-no">2805</span><span id="line-2805"> }</span>
<span class="source-line-no">2806</span><span id="line-2806"></span>
<span class="source-line-no">2807</span><span id="line-2807"> static HbckErrorReporter getErrorReporter(final Configuration conf)</span>
<span class="source-line-no">2808</span><span id="line-2808"> throws ClassNotFoundException {</span>
<span class="source-line-no">2809</span><span id="line-2809"> Class&lt;? extends HbckErrorReporter&gt; reporter = conf.getClass("hbasefsck.errorreporter",</span>
<span class="source-line-no">2810</span><span id="line-2810"> PrintingErrorReporter.class, HbckErrorReporter.class);</span>
<span class="source-line-no">2811</span><span id="line-2811"> return ReflectionUtils.newInstance(reporter, conf);</span>
<span class="source-line-no">2812</span><span id="line-2812"> }</span>
<span class="source-line-no">2813</span><span id="line-2813"></span>
<span class="source-line-no">2814</span><span id="line-2814"> static class PrintingErrorReporter implements HbckErrorReporter {</span>
<span class="source-line-no">2815</span><span id="line-2815"> public int errorCount = 0;</span>
<span class="source-line-no">2816</span><span id="line-2816"> private int showProgress;</span>
<span class="source-line-no">2817</span><span id="line-2817"> // How frequently calls to progress() will create output</span>
<span class="source-line-no">2818</span><span id="line-2818"> private static final int progressThreshold = 100;</span>
<span class="source-line-no">2819</span><span id="line-2819"></span>
<span class="source-line-no">2820</span><span id="line-2820"> Set&lt;HbckTableInfo&gt; errorTables = new HashSet&lt;&gt;();</span>
<span class="source-line-no">2821</span><span id="line-2821"></span>
<span class="source-line-no">2822</span><span id="line-2822"> // for use by unit tests to verify which errors were discovered</span>
<span class="source-line-no">2823</span><span id="line-2823"> private ArrayList&lt;ERROR_CODE&gt; errorList = new ArrayList&lt;&gt;();</span>
<span class="source-line-no">2824</span><span id="line-2824"></span>
<span class="source-line-no">2825</span><span id="line-2825"> @Override</span>
<span class="source-line-no">2826</span><span id="line-2826"> public void clear() {</span>
<span class="source-line-no">2827</span><span id="line-2827"> errorTables.clear();</span>
<span class="source-line-no">2828</span><span id="line-2828"> errorList.clear();</span>
<span class="source-line-no">2829</span><span id="line-2829"> errorCount = 0;</span>
<span class="source-line-no">2830</span><span id="line-2830"> }</span>
<span class="source-line-no">2831</span><span id="line-2831"></span>
<span class="source-line-no">2832</span><span id="line-2832"> @Override</span>
<span class="source-line-no">2833</span><span id="line-2833"> public synchronized void reportError(ERROR_CODE errorCode, String message) {</span>
<span class="source-line-no">2834</span><span id="line-2834"> if (errorCode == ERROR_CODE.WRONG_USAGE) {</span>
<span class="source-line-no">2835</span><span id="line-2835"> System.err.println(message);</span>
<span class="source-line-no">2836</span><span id="line-2836"> return;</span>
<span class="source-line-no">2837</span><span id="line-2837"> }</span>
<span class="source-line-no">2838</span><span id="line-2838"></span>
<span class="source-line-no">2839</span><span id="line-2839"> errorList.add(errorCode);</span>
<span class="source-line-no">2840</span><span id="line-2840"> if (!summary) {</span>
<span class="source-line-no">2841</span><span id="line-2841"> System.out.println("ERROR: " + message);</span>
<span class="source-line-no">2842</span><span id="line-2842"> }</span>
<span class="source-line-no">2843</span><span id="line-2843"> errorCount++;</span>
<span class="source-line-no">2844</span><span id="line-2844"> showProgress = 0;</span>
<span class="source-line-no">2845</span><span id="line-2845"> }</span>
<span class="source-line-no">2846</span><span id="line-2846"></span>
<span class="source-line-no">2847</span><span id="line-2847"> @Override</span>
<span class="source-line-no">2848</span><span id="line-2848"> public synchronized void reportError(ERROR_CODE errorCode, String message,</span>
<span class="source-line-no">2849</span><span id="line-2849"> HbckTableInfo table) {</span>
<span class="source-line-no">2850</span><span id="line-2850"> errorTables.add(table);</span>
<span class="source-line-no">2851</span><span id="line-2851"> reportError(errorCode, message);</span>
<span class="source-line-no">2852</span><span id="line-2852"> }</span>
<span class="source-line-no">2853</span><span id="line-2853"></span>
<span class="source-line-no">2854</span><span id="line-2854"> @Override</span>
<span class="source-line-no">2855</span><span id="line-2855"> public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,</span>
<span class="source-line-no">2856</span><span id="line-2856"> HbckRegionInfo info) {</span>
<span class="source-line-no">2857</span><span id="line-2857"> errorTables.add(table);</span>
<span class="source-line-no">2858</span><span id="line-2858"> String reference = "(region " + info.getRegionNameAsString() + ")";</span>
<span class="source-line-no">2859</span><span id="line-2859"> reportError(errorCode, reference + " " + message);</span>
<span class="source-line-no">2860</span><span id="line-2860"> }</span>
<span class="source-line-no">2861</span><span id="line-2861"></span>
<span class="source-line-no">2862</span><span id="line-2862"> @Override</span>
<span class="source-line-no">2863</span><span id="line-2863"> public synchronized void reportError(ERROR_CODE errorCode, String message, HbckTableInfo table,</span>
<span class="source-line-no">2864</span><span id="line-2864"> HbckRegionInfo info1, HbckRegionInfo info2) {</span>
<span class="source-line-no">2865</span><span id="line-2865"> errorTables.add(table);</span>
<span class="source-line-no">2866</span><span id="line-2866"> String reference =</span>
<span class="source-line-no">2867</span><span id="line-2867"> "(regions " + info1.getRegionNameAsString() + " and " + info2.getRegionNameAsString() + ")";</span>
<span class="source-line-no">2868</span><span id="line-2868"> reportError(errorCode, reference + " " + message);</span>
<span class="source-line-no">2869</span><span id="line-2869"> }</span>
<span class="source-line-no">2870</span><span id="line-2870"></span>
<span class="source-line-no">2871</span><span id="line-2871"> @Override</span>
<span class="source-line-no">2872</span><span id="line-2872"> public synchronized void reportError(String message) {</span>
<span class="source-line-no">2873</span><span id="line-2873"> reportError(ERROR_CODE.UNKNOWN, message);</span>
<span class="source-line-no">2874</span><span id="line-2874"> }</span>
<span class="source-line-no">2875</span><span id="line-2875"></span>
<span class="source-line-no">2876</span><span id="line-2876"> /**</span>
<span class="source-line-no">2877</span><span id="line-2877"> * Report error information, but do not increment the error count. Intended for cases where the</span>
<span class="source-line-no">2878</span><span id="line-2878"> * actual error would have been reported previously.</span>
<span class="source-line-no">2879</span><span id="line-2879"> */</span>
<span class="source-line-no">2880</span><span id="line-2880"> @Override</span>
<span class="source-line-no">2881</span><span id="line-2881"> public synchronized void report(String message) {</span>
<span class="source-line-no">2882</span><span id="line-2882"> if (!summary) {</span>
<span class="source-line-no">2883</span><span id="line-2883"> System.out.println("ERROR: " + message);</span>
<span class="source-line-no">2884</span><span id="line-2884"> }</span>
<span class="source-line-no">2885</span><span id="line-2885"> showProgress = 0;</span>
<span class="source-line-no">2886</span><span id="line-2886"> }</span>
<span class="source-line-no">2887</span><span id="line-2887"></span>
<span class="source-line-no">2888</span><span id="line-2888"> @Override</span>
<span class="source-line-no">2889</span><span id="line-2889"> public synchronized int summarize() {</span>
<span class="source-line-no">2890</span><span id="line-2890"> System.out.println(Integer.toString(errorCount) + " inconsistencies detected.");</span>
<span class="source-line-no">2891</span><span id="line-2891"> if (errorCount == 0) {</span>
<span class="source-line-no">2892</span><span id="line-2892"> System.out.println("Status: OK");</span>
<span class="source-line-no">2893</span><span id="line-2893"> return 0;</span>
<span class="source-line-no">2894</span><span id="line-2894"> } else {</span>
<span class="source-line-no">2895</span><span id="line-2895"> System.out.println("Status: INCONSISTENT");</span>
<span class="source-line-no">2896</span><span id="line-2896"> return -1;</span>
<span class="source-line-no">2897</span><span id="line-2897"> }</span>
<span class="source-line-no">2898</span><span id="line-2898"> }</span>
<span class="source-line-no">2899</span><span id="line-2899"></span>
<span class="source-line-no">2900</span><span id="line-2900"> @Override</span>
<span class="source-line-no">2901</span><span id="line-2901"> public ArrayList&lt;ERROR_CODE&gt; getErrorList() {</span>
<span class="source-line-no">2902</span><span id="line-2902"> return errorList;</span>
<span class="source-line-no">2903</span><span id="line-2903"> }</span>
<span class="source-line-no">2904</span><span id="line-2904"></span>
<span class="source-line-no">2905</span><span id="line-2905"> @Override</span>
<span class="source-line-no">2906</span><span id="line-2906"> public synchronized void print(String message) {</span>
<span class="source-line-no">2907</span><span id="line-2907"> if (!summary) {</span>
<span class="source-line-no">2908</span><span id="line-2908"> System.out.println(message);</span>
<span class="source-line-no">2909</span><span id="line-2909"> }</span>
<span class="source-line-no">2910</span><span id="line-2910"> }</span>
<span class="source-line-no">2911</span><span id="line-2911"></span>
<span class="source-line-no">2912</span><span id="line-2912"> @Override</span>
<span class="source-line-no">2913</span><span id="line-2913"> public boolean tableHasErrors(HbckTableInfo table) {</span>
<span class="source-line-no">2914</span><span id="line-2914"> return errorTables.contains(table);</span>
<span class="source-line-no">2915</span><span id="line-2915"> }</span>
<span class="source-line-no">2916</span><span id="line-2916"></span>
<span class="source-line-no">2917</span><span id="line-2917"> @Override</span>
<span class="source-line-no">2918</span><span id="line-2918"> public void resetErrors() {</span>
<span class="source-line-no">2919</span><span id="line-2919"> errorCount = 0;</span>
<span class="source-line-no">2920</span><span id="line-2920"> }</span>
<span class="source-line-no">2921</span><span id="line-2921"></span>
<span class="source-line-no">2922</span><span id="line-2922"> @Override</span>
<span class="source-line-no">2923</span><span id="line-2923"> public synchronized void detail(String message) {</span>
<span class="source-line-no">2924</span><span id="line-2924"> if (details) {</span>
<span class="source-line-no">2925</span><span id="line-2925"> System.out.println(message);</span>
<span class="source-line-no">2926</span><span id="line-2926"> }</span>
<span class="source-line-no">2927</span><span id="line-2927"> showProgress = 0;</span>
<span class="source-line-no">2928</span><span id="line-2928"> }</span>
<span class="source-line-no">2929</span><span id="line-2929"></span>
<span class="source-line-no">2930</span><span id="line-2930"> @Override</span>
<span class="source-line-no">2931</span><span id="line-2931"> public synchronized void progress() {</span>
<span class="source-line-no">2932</span><span id="line-2932"> if (showProgress++ == progressThreshold) {</span>
<span class="source-line-no">2933</span><span id="line-2933"> if (!summary) {</span>
<span class="source-line-no">2934</span><span id="line-2934"> System.out.print(".");</span>
<span class="source-line-no">2935</span><span id="line-2935"> }</span>
<span class="source-line-no">2936</span><span id="line-2936"> showProgress = 0;</span>
<span class="source-line-no">2937</span><span id="line-2937"> }</span>
<span class="source-line-no">2938</span><span id="line-2938"> }</span>
<span class="source-line-no">2939</span><span id="line-2939"> }</span>
<span class="source-line-no">2940</span><span id="line-2940"></span>
<span class="source-line-no">2941</span><span id="line-2941"> /**</span>
<span class="source-line-no">2942</span><span id="line-2942"> * Contact a region server and get all information from it</span>
<span class="source-line-no">2943</span><span id="line-2943"> */</span>
<span class="source-line-no">2944</span><span id="line-2944"> static class WorkItemRegion implements Callable&lt;Void&gt; {</span>
<span class="source-line-no">2945</span><span id="line-2945"> private final HBaseFsck hbck;</span>
<span class="source-line-no">2946</span><span id="line-2946"> private final ServerName rsinfo;</span>
<span class="source-line-no">2947</span><span id="line-2947"> private final HbckErrorReporter errors;</span>
<span class="source-line-no">2948</span><span id="line-2948"> private final Connection connection;</span>
<span class="source-line-no">2949</span><span id="line-2949"></span>
<span class="source-line-no">2950</span><span id="line-2950"> WorkItemRegion(HBaseFsck hbck, ServerName info, HbckErrorReporter errors,</span>
<span class="source-line-no">2951</span><span id="line-2951"> Connection connection) {</span>
<span class="source-line-no">2952</span><span id="line-2952"> this.hbck = hbck;</span>
<span class="source-line-no">2953</span><span id="line-2953"> this.rsinfo = info;</span>
<span class="source-line-no">2954</span><span id="line-2954"> this.errors = errors;</span>
<span class="source-line-no">2955</span><span id="line-2955"> this.connection = connection;</span>
<span class="source-line-no">2956</span><span id="line-2956"> }</span>
<span class="source-line-no">2957</span><span id="line-2957"></span>
<span class="source-line-no">2958</span><span id="line-2958"> @Override</span>
<span class="source-line-no">2959</span><span id="line-2959"> public synchronized Void call() throws IOException {</span>
<span class="source-line-no">2960</span><span id="line-2960"> errors.progress();</span>
<span class="source-line-no">2961</span><span id="line-2961"> try {</span>
<span class="source-line-no">2962</span><span id="line-2962"> // list all online regions from this region server</span>
<span class="source-line-no">2963</span><span id="line-2963"> List&lt;RegionInfo&gt; regions = connection.getAdmin().getRegions(rsinfo);</span>
<span class="source-line-no">2964</span><span id="line-2964"> regions = filterRegions(regions);</span>
<span class="source-line-no">2965</span><span id="line-2965"></span>
<span class="source-line-no">2966</span><span id="line-2966"> if (details) {</span>
<span class="source-line-no">2967</span><span id="line-2967"> errors.detail(</span>
<span class="source-line-no">2968</span><span id="line-2968"> "RegionServer: " + rsinfo.getServerName() + " number of regions: " + regions.size());</span>
<span class="source-line-no">2969</span><span id="line-2969"> for (RegionInfo rinfo : regions) {</span>
<span class="source-line-no">2970</span><span id="line-2970"> errors.detail(" " + rinfo.getRegionNameAsString() + " id: " + rinfo.getRegionId()</span>
<span class="source-line-no">2971</span><span id="line-2971"> + " encoded_name: " + rinfo.getEncodedName() + " start: "</span>
<span class="source-line-no">2972</span><span id="line-2972"> + Bytes.toStringBinary(rinfo.getStartKey()) + " end: "</span>
<span class="source-line-no">2973</span><span id="line-2973"> + Bytes.toStringBinary(rinfo.getEndKey()));</span>
<span class="source-line-no">2974</span><span id="line-2974"> }</span>
<span class="source-line-no">2975</span><span id="line-2975"> }</span>
<span class="source-line-no">2976</span><span id="line-2976"></span>
<span class="source-line-no">2977</span><span id="line-2977"> // check to see if the existence of this region matches the region in META</span>
<span class="source-line-no">2978</span><span id="line-2978"> for (RegionInfo r : regions) {</span>
<span class="source-line-no">2979</span><span id="line-2979"> HbckRegionInfo hbi = hbck.getOrCreateInfo(r.getEncodedName());</span>
<span class="source-line-no">2980</span><span id="line-2980"> hbi.addServer(r, rsinfo);</span>
<span class="source-line-no">2981</span><span id="line-2981"> }</span>
<span class="source-line-no">2982</span><span id="line-2982"> } catch (IOException e) { // unable to connect to the region server.</span>
<span class="source-line-no">2983</span><span id="line-2983"> errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE,</span>
<span class="source-line-no">2984</span><span id="line-2984"> "RegionServer: " + rsinfo.getServerName() + " Unable to fetch region information. " + e);</span>
<span class="source-line-no">2985</span><span id="line-2985"> throw e;</span>
<span class="source-line-no">2986</span><span id="line-2986"> }</span>
<span class="source-line-no">2987</span><span id="line-2987"> return null;</span>
<span class="source-line-no">2988</span><span id="line-2988"> }</span>
<span class="source-line-no">2989</span><span id="line-2989"></span>
<span class="source-line-no">2990</span><span id="line-2990"> private List&lt;RegionInfo&gt; filterRegions(List&lt;RegionInfo&gt; regions) {</span>
<span class="source-line-no">2991</span><span id="line-2991"> List&lt;RegionInfo&gt; ret = Lists.newArrayList();</span>
<span class="source-line-no">2992</span><span id="line-2992"> for (RegionInfo hri : regions) {</span>
<span class="source-line-no">2993</span><span id="line-2993"> if (hri.isMetaRegion() || (!hbck.checkMetaOnly &amp;&amp; hbck.isTableIncluded(hri.getTable()))) {</span>
<span class="source-line-no">2994</span><span id="line-2994"> ret.add(hri);</span>
<span class="source-line-no">2995</span><span id="line-2995"> }</span>
<span class="source-line-no">2996</span><span id="line-2996"> }</span>
<span class="source-line-no">2997</span><span id="line-2997"> return ret;</span>
<span class="source-line-no">2998</span><span id="line-2998"> }</span>
<span class="source-line-no">2999</span><span id="line-2999"> }</span>
<span class="source-line-no">3000</span><span id="line-3000"></span>
<span class="source-line-no">3001</span><span id="line-3001"> /**</span>
<span class="source-line-no">3002</span><span id="line-3002"> * Contact hdfs and get all information about specified table directory into regioninfo list.</span>
<span class="source-line-no">3003</span><span id="line-3003"> */</span>
<span class="source-line-no">3004</span><span id="line-3004"> class WorkItemHdfsDir implements Callable&lt;Void&gt; {</span>
<span class="source-line-no">3005</span><span id="line-3005"> private FileStatus tableDir;</span>
<span class="source-line-no">3006</span><span id="line-3006"> private HbckErrorReporter errors;</span>
<span class="source-line-no">3007</span><span id="line-3007"> private FileSystem fs;</span>
<span class="source-line-no">3008</span><span id="line-3008"></span>
<span class="source-line-no">3009</span><span id="line-3009"> WorkItemHdfsDir(FileSystem fs, HbckErrorReporter errors, FileStatus status) {</span>
<span class="source-line-no">3010</span><span id="line-3010"> this.fs = fs;</span>
<span class="source-line-no">3011</span><span id="line-3011"> this.tableDir = status;</span>
<span class="source-line-no">3012</span><span id="line-3012"> this.errors = errors;</span>
<span class="source-line-no">3013</span><span id="line-3013"> }</span>
<span class="source-line-no">3014</span><span id="line-3014"></span>
<span class="source-line-no">3015</span><span id="line-3015"> @Override</span>
<span class="source-line-no">3016</span><span id="line-3016"> public synchronized Void call() throws InterruptedException, ExecutionException {</span>
<span class="source-line-no">3017</span><span id="line-3017"> final Vector&lt;Exception&gt; exceptions = new Vector&lt;&gt;();</span>
<span class="source-line-no">3018</span><span id="line-3018"></span>
<span class="source-line-no">3019</span><span id="line-3019"> try {</span>
<span class="source-line-no">3020</span><span id="line-3020"> final FileStatus[] regionDirs = fs.listStatus(tableDir.getPath());</span>
<span class="source-line-no">3021</span><span id="line-3021"> final List&lt;Future&lt;?&gt;&gt; futures = new ArrayList&lt;&gt;(regionDirs.length);</span>
<span class="source-line-no">3022</span><span id="line-3022"></span>
<span class="source-line-no">3023</span><span id="line-3023"> for (final FileStatus regionDir : regionDirs) {</span>
<span class="source-line-no">3024</span><span id="line-3024"> errors.progress();</span>
<span class="source-line-no">3025</span><span id="line-3025"> final String encodedName = regionDir.getPath().getName();</span>
<span class="source-line-no">3026</span><span id="line-3026"> // ignore directories that aren't hexadecimal</span>
<span class="source-line-no">3027</span><span id="line-3027"> if (!encodedName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {</span>
<span class="source-line-no">3028</span><span id="line-3028"> continue;</span>
<span class="source-line-no">3029</span><span id="line-3029"> }</span>
<span class="source-line-no">3030</span><span id="line-3030"></span>
<span class="source-line-no">3031</span><span id="line-3031"> if (!exceptions.isEmpty()) {</span>
<span class="source-line-no">3032</span><span id="line-3032"> break;</span>
<span class="source-line-no">3033</span><span id="line-3033"> }</span>
<span class="source-line-no">3034</span><span id="line-3034"></span>
<span class="source-line-no">3035</span><span id="line-3035"> futures.add(executor.submit(new Runnable() {</span>
<span class="source-line-no">3036</span><span id="line-3036"> @Override</span>
<span class="source-line-no">3037</span><span id="line-3037"> public void run() {</span>
<span class="source-line-no">3038</span><span id="line-3038"> try {</span>
<span class="source-line-no">3039</span><span id="line-3039"> LOG.debug("Loading region info from hdfs:" + regionDir.getPath());</span>
<span class="source-line-no">3040</span><span id="line-3040"></span>
<span class="source-line-no">3041</span><span id="line-3041"> Path regioninfoFile =</span>
<span class="source-line-no">3042</span><span id="line-3042"> new Path(regionDir.getPath(), HRegionFileSystem.REGION_INFO_FILE);</span>
<span class="source-line-no">3043</span><span id="line-3043"> boolean regioninfoFileExists = fs.exists(regioninfoFile);</span>
<span class="source-line-no">3044</span><span id="line-3044"></span>
<span class="source-line-no">3045</span><span id="line-3045"> if (!regioninfoFileExists) {</span>
<span class="source-line-no">3046</span><span id="line-3046"> // As tables become larger it is more and more likely that by the time you</span>
<span class="source-line-no">3047</span><span id="line-3047"> // reach a given region that it will be gone due to region splits/merges.</span>
<span class="source-line-no">3048</span><span id="line-3048"> if (!fs.exists(regionDir.getPath())) {</span>
<span class="source-line-no">3049</span><span id="line-3049"> LOG.warn("By the time we tried to process this region dir it was already gone: "</span>
<span class="source-line-no">3050</span><span id="line-3050"> + regionDir.getPath());</span>
<span class="source-line-no">3051</span><span id="line-3051"> return;</span>
<span class="source-line-no">3052</span><span id="line-3052"> }</span>
<span class="source-line-no">3053</span><span id="line-3053"> }</span>
<span class="source-line-no">3054</span><span id="line-3054"></span>
<span class="source-line-no">3055</span><span id="line-3055"> HbckRegionInfo hbi = HBaseFsck.this.getOrCreateInfo(encodedName);</span>
<span class="source-line-no">3056</span><span id="line-3056"> HbckRegionInfo.HdfsEntry he = new HbckRegionInfo.HdfsEntry();</span>
<span class="source-line-no">3057</span><span id="line-3057"> synchronized (hbi) {</span>
<span class="source-line-no">3058</span><span id="line-3058"> if (hbi.getHdfsRegionDir() != null) {</span>
<span class="source-line-no">3059</span><span id="line-3059"> errors</span>
<span class="source-line-no">3060</span><span id="line-3060"> .print("Directory " + encodedName + " duplicate??" + hbi.getHdfsRegionDir());</span>
<span class="source-line-no">3061</span><span id="line-3061"> }</span>
<span class="source-line-no">3062</span><span id="line-3062"></span>
<span class="source-line-no">3063</span><span id="line-3063"> he.regionDir = regionDir.getPath();</span>
<span class="source-line-no">3064</span><span id="line-3064"> he.regionDirModTime = regionDir.getModificationTime();</span>
<span class="source-line-no">3065</span><span id="line-3065"> he.hdfsRegioninfoFilePresent = regioninfoFileExists;</span>
<span class="source-line-no">3066</span><span id="line-3066"> // we add to orphan list when we attempt to read .regioninfo</span>
<span class="source-line-no">3067</span><span id="line-3067"></span>
<span class="source-line-no">3068</span><span id="line-3068"> // Set a flag if this region contains only edits</span>
<span class="source-line-no">3069</span><span id="line-3069"> // This is special case if a region is left after split</span>
<span class="source-line-no">3070</span><span id="line-3070"> he.hdfsOnlyEdits = true;</span>
<span class="source-line-no">3071</span><span id="line-3071"> FileStatus[] subDirs = fs.listStatus(regionDir.getPath());</span>
<span class="source-line-no">3072</span><span id="line-3072"> Path ePath = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir.getPath());</span>
<span class="source-line-no">3073</span><span id="line-3073"> for (FileStatus subDir : subDirs) {</span>
<span class="source-line-no">3074</span><span id="line-3074"> errors.progress();</span>
<span class="source-line-no">3075</span><span id="line-3075"> String sdName = subDir.getPath().getName();</span>
<span class="source-line-no">3076</span><span id="line-3076"> if (!sdName.startsWith(".") &amp;&amp; !sdName.equals(ePath.getName())) {</span>
<span class="source-line-no">3077</span><span id="line-3077"> he.hdfsOnlyEdits = false;</span>
<span class="source-line-no">3078</span><span id="line-3078"> break;</span>
<span class="source-line-no">3079</span><span id="line-3079"> }</span>
<span class="source-line-no">3080</span><span id="line-3080"> }</span>
<span class="source-line-no">3081</span><span id="line-3081"> hbi.setHdfsEntry(he);</span>
<span class="source-line-no">3082</span><span id="line-3082"> }</span>
<span class="source-line-no">3083</span><span id="line-3083"> } catch (Exception e) {</span>
<span class="source-line-no">3084</span><span id="line-3084"> LOG.error("Could not load region dir", e);</span>
<span class="source-line-no">3085</span><span id="line-3085"> exceptions.add(e);</span>
<span class="source-line-no">3086</span><span id="line-3086"> }</span>
<span class="source-line-no">3087</span><span id="line-3087"> }</span>
<span class="source-line-no">3088</span><span id="line-3088"> }));</span>
<span class="source-line-no">3089</span><span id="line-3089"> }</span>
<span class="source-line-no">3090</span><span id="line-3090"></span>
<span class="source-line-no">3091</span><span id="line-3091"> // Ensure all pending tasks are complete (or that we run into an exception)</span>
<span class="source-line-no">3092</span><span id="line-3092"> for (Future&lt;?&gt; f : futures) {</span>
<span class="source-line-no">3093</span><span id="line-3093"> if (!exceptions.isEmpty()) {</span>
<span class="source-line-no">3094</span><span id="line-3094"> break;</span>
<span class="source-line-no">3095</span><span id="line-3095"> }</span>
<span class="source-line-no">3096</span><span id="line-3096"> try {</span>
<span class="source-line-no">3097</span><span id="line-3097"> f.get();</span>
<span class="source-line-no">3098</span><span id="line-3098"> } catch (ExecutionException e) {</span>
<span class="source-line-no">3099</span><span id="line-3099"> LOG.error("Unexpected exec exception! Should've been caught already. (Bug?)", e);</span>
<span class="source-line-no">3100</span><span id="line-3100"> // Shouldn't happen, we already logged/caught any exceptions in the Runnable</span>
<span class="source-line-no">3101</span><span id="line-3101"> }</span>
<span class="source-line-no">3102</span><span id="line-3102"> }</span>
<span class="source-line-no">3103</span><span id="line-3103"> } catch (IOException e) {</span>
<span class="source-line-no">3104</span><span id="line-3104"> LOG.error("Cannot execute WorkItemHdfsDir for " + tableDir, e);</span>
<span class="source-line-no">3105</span><span id="line-3105"> exceptions.add(e);</span>
<span class="source-line-no">3106</span><span id="line-3106"> } finally {</span>
<span class="source-line-no">3107</span><span id="line-3107"> if (!exceptions.isEmpty()) {</span>
<span class="source-line-no">3108</span><span id="line-3108"> errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "Table Directory: "</span>
<span class="source-line-no">3109</span><span id="line-3109"> + tableDir.getPath().getName() + " Unable to fetch all HDFS region information. ");</span>
<span class="source-line-no">3110</span><span id="line-3110"> // Just throw the first exception as an indication something bad happened</span>
<span class="source-line-no">3111</span><span id="line-3111"> // Don't need to propagate all the exceptions, we already logged them all anyway</span>
<span class="source-line-no">3112</span><span id="line-3112"> throw new ExecutionException("First exception in WorkItemHdfsDir",</span>
<span class="source-line-no">3113</span><span id="line-3113"> exceptions.firstElement());</span>
<span class="source-line-no">3114</span><span id="line-3114"> }</span>
<span class="source-line-no">3115</span><span id="line-3115"> }</span>
<span class="source-line-no">3116</span><span id="line-3116"> return null;</span>
<span class="source-line-no">3117</span><span id="line-3117"> }</span>
<span class="source-line-no">3118</span><span id="line-3118"> }</span>
<span class="source-line-no">3119</span><span id="line-3119"></span>
<span class="source-line-no">3120</span><span id="line-3120"> /**</span>
<span class="source-line-no">3121</span><span id="line-3121"> * Contact hdfs and get all information about specified table directory into regioninfo list.</span>
<span class="source-line-no">3122</span><span id="line-3122"> */</span>
<span class="source-line-no">3123</span><span id="line-3123"> static class WorkItemHdfsRegionInfo implements Callable&lt;Void&gt; {</span>
<span class="source-line-no">3124</span><span id="line-3124"> private HbckRegionInfo hbi;</span>
<span class="source-line-no">3125</span><span id="line-3125"> private HBaseFsck hbck;</span>
<span class="source-line-no">3126</span><span id="line-3126"> private HbckErrorReporter errors;</span>
<span class="source-line-no">3127</span><span id="line-3127"></span>
<span class="source-line-no">3128</span><span id="line-3128"> WorkItemHdfsRegionInfo(HbckRegionInfo hbi, HBaseFsck hbck, HbckErrorReporter errors) {</span>
<span class="source-line-no">3129</span><span id="line-3129"> this.hbi = hbi;</span>
<span class="source-line-no">3130</span><span id="line-3130"> this.hbck = hbck;</span>
<span class="source-line-no">3131</span><span id="line-3131"> this.errors = errors;</span>
<span class="source-line-no">3132</span><span id="line-3132"> }</span>
<span class="source-line-no">3133</span><span id="line-3133"></span>
<span class="source-line-no">3134</span><span id="line-3134"> @Override</span>
<span class="source-line-no">3135</span><span id="line-3135"> public synchronized Void call() throws IOException {</span>
<span class="source-line-no">3136</span><span id="line-3136"> // only load entries that haven't been loaded yet.</span>
<span class="source-line-no">3137</span><span id="line-3137"> if (hbi.getHdfsHRI() == null) {</span>
<span class="source-line-no">3138</span><span id="line-3138"> try {</span>
<span class="source-line-no">3139</span><span id="line-3139"> errors.progress();</span>
<span class="source-line-no">3140</span><span id="line-3140"> hbi.loadHdfsRegioninfo(hbck.getConf());</span>
<span class="source-line-no">3141</span><span id="line-3141"> } catch (IOException ioe) {</span>
<span class="source-line-no">3142</span><span id="line-3142"> String msg = "Orphan region in HDFS: Unable to load .regioninfo from table "</span>
<span class="source-line-no">3143</span><span id="line-3143"> + hbi.getTableName() + " in hdfs dir " + hbi.getHdfsRegionDir()</span>
<span class="source-line-no">3144</span><span id="line-3144"> + "! It may be an invalid format or version file. Treating as "</span>
<span class="source-line-no">3145</span><span id="line-3145"> + "an orphaned regiondir.";</span>
<span class="source-line-no">3146</span><span id="line-3146"> errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg);</span>
<span class="source-line-no">3147</span><span id="line-3147"> try {</span>
<span class="source-line-no">3148</span><span id="line-3148"> hbck.debugLsr(hbi.getHdfsRegionDir());</span>
<span class="source-line-no">3149</span><span id="line-3149"> } catch (IOException ioe2) {</span>
<span class="source-line-no">3150</span><span id="line-3150"> LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2);</span>
<span class="source-line-no">3151</span><span id="line-3151"> throw ioe2;</span>
<span class="source-line-no">3152</span><span id="line-3152"> }</span>
<span class="source-line-no">3153</span><span id="line-3153"> hbck.orphanHdfsDirs.add(hbi);</span>
<span class="source-line-no">3154</span><span id="line-3154"> throw ioe;</span>
<span class="source-line-no">3155</span><span id="line-3155"> }</span>
<span class="source-line-no">3156</span><span id="line-3156"> }</span>
<span class="source-line-no">3157</span><span id="line-3157"> return null;</span>
<span class="source-line-no">3158</span><span id="line-3158"> }</span>
<span class="source-line-no">3159</span><span id="line-3159"> }</span>
<span class="source-line-no">3160</span><span id="line-3160"></span>
<span class="source-line-no">3161</span><span id="line-3161"> /**</span>
<span class="source-line-no">3162</span><span id="line-3162"> * Display the full report from fsck. This displays all live and dead region servers, and all</span>
<span class="source-line-no">3163</span><span id="line-3163"> * known regions.</span>
<span class="source-line-no">3164</span><span id="line-3164"> */</span>
<span class="source-line-no">3165</span><span id="line-3165"> public static void setDisplayFullReport() {</span>
<span class="source-line-no">3166</span><span id="line-3166"> details = true;</span>
<span class="source-line-no">3167</span><span id="line-3167"> }</span>
<span class="source-line-no">3168</span><span id="line-3168"></span>
<span class="source-line-no">3169</span><span id="line-3169"> public static boolean shouldDisplayFullReport() {</span>
<span class="source-line-no">3170</span><span id="line-3170"> return details;</span>
<span class="source-line-no">3171</span><span id="line-3171"> }</span>
<span class="source-line-no">3172</span><span id="line-3172"></span>
<span class="source-line-no">3173</span><span id="line-3173"> /**</span>
<span class="source-line-no">3174</span><span id="line-3174"> * Set exclusive mode.</span>
<span class="source-line-no">3175</span><span id="line-3175"> */</span>
<span class="source-line-no">3176</span><span id="line-3176"> public static void setForceExclusive() {</span>
<span class="source-line-no">3177</span><span id="line-3177"> forceExclusive = true;</span>
<span class="source-line-no">3178</span><span id="line-3178"> }</span>
<span class="source-line-no">3179</span><span id="line-3179"></span>
<span class="source-line-no">3180</span><span id="line-3180"> /**</span>
<span class="source-line-no">3181</span><span id="line-3181"> * Only one instance of hbck can modify HBase at a time.</span>
<span class="source-line-no">3182</span><span id="line-3182"> */</span>
<span class="source-line-no">3183</span><span id="line-3183"> public boolean isExclusive() {</span>
<span class="source-line-no">3184</span><span id="line-3184"> return fixAny || forceExclusive;</span>
<span class="source-line-no">3185</span><span id="line-3185"> }</span>
<span class="source-line-no">3186</span><span id="line-3186"></span>
<span class="source-line-no">3187</span><span id="line-3187"> /**</span>
<span class="source-line-no">3188</span><span id="line-3188"> * Set summary mode. Print only summary of the tables and status (OK or INCONSISTENT)</span>
<span class="source-line-no">3189</span><span id="line-3189"> */</span>
<span class="source-line-no">3190</span><span id="line-3190"> static void setSummary() {</span>
<span class="source-line-no">3191</span><span id="line-3191"> summary = true;</span>
<span class="source-line-no">3192</span><span id="line-3192"> }</span>
<span class="source-line-no">3193</span><span id="line-3193"></span>
<span class="source-line-no">3194</span><span id="line-3194"> /**</span>
<span class="source-line-no">3195</span><span id="line-3195"> * Set hbase:meta check mode. Print only info about hbase:meta table deployment/state</span>
<span class="source-line-no">3196</span><span id="line-3196"> */</span>
<span class="source-line-no">3197</span><span id="line-3197"> void setCheckMetaOnly() {</span>
<span class="source-line-no">3198</span><span id="line-3198"> checkMetaOnly = true;</span>
<span class="source-line-no">3199</span><span id="line-3199"> }</span>
<span class="source-line-no">3200</span><span id="line-3200"></span>
<span class="source-line-no">3201</span><span id="line-3201"> /**</span>
<span class="source-line-no">3202</span><span id="line-3202"> * Set region boundaries check mode.</span>
<span class="source-line-no">3203</span><span id="line-3203"> */</span>
<span class="source-line-no">3204</span><span id="line-3204"> void setRegionBoundariesCheck() {</span>
<span class="source-line-no">3205</span><span id="line-3205"> checkRegionBoundaries = true;</span>
<span class="source-line-no">3206</span><span id="line-3206"> }</span>
<span class="source-line-no">3207</span><span id="line-3207"></span>
<span class="source-line-no">3208</span><span id="line-3208"> /**</span>
<span class="source-line-no">3209</span><span id="line-3209"> * Set replication fix mode.</span>
<span class="source-line-no">3210</span><span id="line-3210"> */</span>
<span class="source-line-no">3211</span><span id="line-3211"> public void setFixReplication(boolean shouldFix) {</span>
<span class="source-line-no">3212</span><span id="line-3212"> fixReplication = shouldFix;</span>
<span class="source-line-no">3213</span><span id="line-3213"> fixAny |= shouldFix;</span>
<span class="source-line-no">3214</span><span id="line-3214"> }</span>
<span class="source-line-no">3215</span><span id="line-3215"></span>
<span class="source-line-no">3216</span><span id="line-3216"> public void setCleanReplicationBarrier(boolean shouldClean) {</span>
<span class="source-line-no">3217</span><span id="line-3217"> cleanReplicationBarrier = shouldClean;</span>
<span class="source-line-no">3218</span><span id="line-3218"> }</span>
<span class="source-line-no">3219</span><span id="line-3219"></span>
<span class="source-line-no">3220</span><span id="line-3220"> /**</span>
<span class="source-line-no">3221</span><span id="line-3221"> * Check if we should rerun fsck again. This checks if we've tried to fix something and we should</span>
<span class="source-line-no">3222</span><span id="line-3222"> * rerun fsck tool again. Display the full report from fsck. This displays all live and dead</span>
<span class="source-line-no">3223</span><span id="line-3223"> * region servers, and all known regions.</span>
<span class="source-line-no">3224</span><span id="line-3224"> */</span>
<span class="source-line-no">3225</span><span id="line-3225"> void setShouldRerun() {</span>
<span class="source-line-no">3226</span><span id="line-3226"> rerun = true;</span>
<span class="source-line-no">3227</span><span id="line-3227"> }</span>
<span class="source-line-no">3228</span><span id="line-3228"></span>
<span class="source-line-no">3229</span><span id="line-3229"> public boolean shouldRerun() {</span>
<span class="source-line-no">3230</span><span id="line-3230"> return rerun;</span>
<span class="source-line-no">3231</span><span id="line-3231"> }</span>
<span class="source-line-no">3232</span><span id="line-3232"></span>
<span class="source-line-no">3233</span><span id="line-3233"> /**</span>
<span class="source-line-no">3234</span><span id="line-3234"> * Fix inconsistencies found by fsck. This should try to fix errors (if any) found by fsck</span>
<span class="source-line-no">3235</span><span id="line-3235"> * utility.</span>
<span class="source-line-no">3236</span><span id="line-3236"> */</span>
<span class="source-line-no">3237</span><span id="line-3237"> public void setFixAssignments(boolean shouldFix) {</span>
<span class="source-line-no">3238</span><span id="line-3238"> fixAssignments = shouldFix;</span>
<span class="source-line-no">3239</span><span id="line-3239"> fixAny |= shouldFix;</span>
<span class="source-line-no">3240</span><span id="line-3240"> }</span>
<span class="source-line-no">3241</span><span id="line-3241"></span>
<span class="source-line-no">3242</span><span id="line-3242"> boolean shouldFixAssignments() {</span>
<span class="source-line-no">3243</span><span id="line-3243"> return fixAssignments;</span>
<span class="source-line-no">3244</span><span id="line-3244"> }</span>
<span class="source-line-no">3245</span><span id="line-3245"></span>
<span class="source-line-no">3246</span><span id="line-3246"> public void setFixMeta(boolean shouldFix) {</span>
<span class="source-line-no">3247</span><span id="line-3247"> fixMeta = shouldFix;</span>
<span class="source-line-no">3248</span><span id="line-3248"> fixAny |= shouldFix;</span>
<span class="source-line-no">3249</span><span id="line-3249"> }</span>
<span class="source-line-no">3250</span><span id="line-3250"></span>
<span class="source-line-no">3251</span><span id="line-3251"> boolean shouldFixMeta() {</span>
<span class="source-line-no">3252</span><span id="line-3252"> return fixMeta;</span>
<span class="source-line-no">3253</span><span id="line-3253"> }</span>
<span class="source-line-no">3254</span><span id="line-3254"></span>
<span class="source-line-no">3255</span><span id="line-3255"> public void setFixEmptyMetaCells(boolean shouldFix) {</span>
<span class="source-line-no">3256</span><span id="line-3256"> fixEmptyMetaCells = shouldFix;</span>
<span class="source-line-no">3257</span><span id="line-3257"> fixAny |= shouldFix;</span>
<span class="source-line-no">3258</span><span id="line-3258"> }</span>
<span class="source-line-no">3259</span><span id="line-3259"></span>
<span class="source-line-no">3260</span><span id="line-3260"> boolean shouldFixEmptyMetaCells() {</span>
<span class="source-line-no">3261</span><span id="line-3261"> return fixEmptyMetaCells;</span>
<span class="source-line-no">3262</span><span id="line-3262"> }</span>
<span class="source-line-no">3263</span><span id="line-3263"></span>
<span class="source-line-no">3264</span><span id="line-3264"> public void setCheckHdfs(boolean checking) {</span>
<span class="source-line-no">3265</span><span id="line-3265"> checkHdfs = checking;</span>
<span class="source-line-no">3266</span><span id="line-3266"> }</span>
<span class="source-line-no">3267</span><span id="line-3267"></span>
<span class="source-line-no">3268</span><span id="line-3268"> boolean shouldCheckHdfs() {</span>
<span class="source-line-no">3269</span><span id="line-3269"> return checkHdfs;</span>
<span class="source-line-no">3270</span><span id="line-3270"> }</span>
<span class="source-line-no">3271</span><span id="line-3271"></span>
<span class="source-line-no">3272</span><span id="line-3272"> public void setFixHdfsHoles(boolean shouldFix) {</span>
<span class="source-line-no">3273</span><span id="line-3273"> fixHdfsHoles = shouldFix;</span>
<span class="source-line-no">3274</span><span id="line-3274"> fixAny |= shouldFix;</span>
<span class="source-line-no">3275</span><span id="line-3275"> }</span>
<span class="source-line-no">3276</span><span id="line-3276"></span>
<span class="source-line-no">3277</span><span id="line-3277"> boolean shouldFixHdfsHoles() {</span>
<span class="source-line-no">3278</span><span id="line-3278"> return fixHdfsHoles;</span>
<span class="source-line-no">3279</span><span id="line-3279"> }</span>
<span class="source-line-no">3280</span><span id="line-3280"></span>
<span class="source-line-no">3281</span><span id="line-3281"> public void setFixTableOrphans(boolean shouldFix) {</span>
<span class="source-line-no">3282</span><span id="line-3282"> fixTableOrphans = shouldFix;</span>
<span class="source-line-no">3283</span><span id="line-3283"> fixAny |= shouldFix;</span>
<span class="source-line-no">3284</span><span id="line-3284"> }</span>
<span class="source-line-no">3285</span><span id="line-3285"></span>
<span class="source-line-no">3286</span><span id="line-3286"> boolean shouldFixTableOrphans() {</span>
<span class="source-line-no">3287</span><span id="line-3287"> return fixTableOrphans;</span>
<span class="source-line-no">3288</span><span id="line-3288"> }</span>
<span class="source-line-no">3289</span><span id="line-3289"></span>
<span class="source-line-no">3290</span><span id="line-3290"> public void setFixHdfsOverlaps(boolean shouldFix) {</span>
<span class="source-line-no">3291</span><span id="line-3291"> fixHdfsOverlaps = shouldFix;</span>
<span class="source-line-no">3292</span><span id="line-3292"> fixAny |= shouldFix;</span>
<span class="source-line-no">3293</span><span id="line-3293"> }</span>
<span class="source-line-no">3294</span><span id="line-3294"></span>
<span class="source-line-no">3295</span><span id="line-3295"> boolean shouldFixHdfsOverlaps() {</span>
<span class="source-line-no">3296</span><span id="line-3296"> return fixHdfsOverlaps;</span>
<span class="source-line-no">3297</span><span id="line-3297"> }</span>
<span class="source-line-no">3298</span><span id="line-3298"></span>
<span class="source-line-no">3299</span><span id="line-3299"> public void setFixHdfsOrphans(boolean shouldFix) {</span>
<span class="source-line-no">3300</span><span id="line-3300"> fixHdfsOrphans = shouldFix;</span>
<span class="source-line-no">3301</span><span id="line-3301"> fixAny |= shouldFix;</span>
<span class="source-line-no">3302</span><span id="line-3302"> }</span>
<span class="source-line-no">3303</span><span id="line-3303"></span>
<span class="source-line-no">3304</span><span id="line-3304"> boolean shouldFixHdfsOrphans() {</span>
<span class="source-line-no">3305</span><span id="line-3305"> return fixHdfsOrphans;</span>
<span class="source-line-no">3306</span><span id="line-3306"> }</span>
<span class="source-line-no">3307</span><span id="line-3307"></span>
<span class="source-line-no">3308</span><span id="line-3308"> public void setFixVersionFile(boolean shouldFix) {</span>
<span class="source-line-no">3309</span><span id="line-3309"> fixVersionFile = shouldFix;</span>
<span class="source-line-no">3310</span><span id="line-3310"> fixAny |= shouldFix;</span>
<span class="source-line-no">3311</span><span id="line-3311"> }</span>
<span class="source-line-no">3312</span><span id="line-3312"></span>
<span class="source-line-no">3313</span><span id="line-3313"> public boolean shouldFixVersionFile() {</span>
<span class="source-line-no">3314</span><span id="line-3314"> return fixVersionFile;</span>
<span class="source-line-no">3315</span><span id="line-3315"> }</span>
<span class="source-line-no">3316</span><span id="line-3316"></span>
<span class="source-line-no">3317</span><span id="line-3317"> public void setSidelineBigOverlaps(boolean sbo) {</span>
<span class="source-line-no">3318</span><span id="line-3318"> this.sidelineBigOverlaps = sbo;</span>
<span class="source-line-no">3319</span><span id="line-3319"> }</span>
<span class="source-line-no">3320</span><span id="line-3320"></span>
<span class="source-line-no">3321</span><span id="line-3321"> public boolean shouldSidelineBigOverlaps() {</span>
<span class="source-line-no">3322</span><span id="line-3322"> return sidelineBigOverlaps;</span>
<span class="source-line-no">3323</span><span id="line-3323"> }</span>
<span class="source-line-no">3324</span><span id="line-3324"></span>
<span class="source-line-no">3325</span><span id="line-3325"> public void setFixSplitParents(boolean shouldFix) {</span>
<span class="source-line-no">3326</span><span id="line-3326"> fixSplitParents = shouldFix;</span>
<span class="source-line-no">3327</span><span id="line-3327"> fixAny |= shouldFix;</span>
<span class="source-line-no">3328</span><span id="line-3328"> }</span>
<span class="source-line-no">3329</span><span id="line-3329"></span>
<span class="source-line-no">3330</span><span id="line-3330"> public void setRemoveParents(boolean shouldFix) {</span>
<span class="source-line-no">3331</span><span id="line-3331"> removeParents = shouldFix;</span>
<span class="source-line-no">3332</span><span id="line-3332"> fixAny |= shouldFix;</span>
<span class="source-line-no">3333</span><span id="line-3333"> }</span>
<span class="source-line-no">3334</span><span id="line-3334"></span>
<span class="source-line-no">3335</span><span id="line-3335"> boolean shouldFixSplitParents() {</span>
<span class="source-line-no">3336</span><span id="line-3336"> return fixSplitParents;</span>
<span class="source-line-no">3337</span><span id="line-3337"> }</span>
<span class="source-line-no">3338</span><span id="line-3338"></span>
<span class="source-line-no">3339</span><span id="line-3339"> boolean shouldRemoveParents() {</span>
<span class="source-line-no">3340</span><span id="line-3340"> return removeParents;</span>
<span class="source-line-no">3341</span><span id="line-3341"> }</span>
<span class="source-line-no">3342</span><span id="line-3342"></span>
<span class="source-line-no">3343</span><span id="line-3343"> public void setFixReferenceFiles(boolean shouldFix) {</span>
<span class="source-line-no">3344</span><span id="line-3344"> fixReferenceFiles = shouldFix;</span>
<span class="source-line-no">3345</span><span id="line-3345"> fixAny |= shouldFix;</span>
<span class="source-line-no">3346</span><span id="line-3346"> }</span>
<span class="source-line-no">3347</span><span id="line-3347"></span>
<span class="source-line-no">3348</span><span id="line-3348"> boolean shouldFixReferenceFiles() {</span>
<span class="source-line-no">3349</span><span id="line-3349"> return fixReferenceFiles;</span>
<span class="source-line-no">3350</span><span id="line-3350"> }</span>
<span class="source-line-no">3351</span><span id="line-3351"></span>
<span class="source-line-no">3352</span><span id="line-3352"> public void setFixHFileLinks(boolean shouldFix) {</span>
<span class="source-line-no">3353</span><span id="line-3353"> fixHFileLinks = shouldFix;</span>
<span class="source-line-no">3354</span><span id="line-3354"> fixAny |= shouldFix;</span>
<span class="source-line-no">3355</span><span id="line-3355"> }</span>
<span class="source-line-no">3356</span><span id="line-3356"></span>
<span class="source-line-no">3357</span><span id="line-3357"> boolean shouldFixHFileLinks() {</span>
<span class="source-line-no">3358</span><span id="line-3358"> return fixHFileLinks;</span>
<span class="source-line-no">3359</span><span id="line-3359"> }</span>
<span class="source-line-no">3360</span><span id="line-3360"></span>
<span class="source-line-no">3361</span><span id="line-3361"> public boolean shouldIgnorePreCheckPermission() {</span>
<span class="source-line-no">3362</span><span id="line-3362"> return !fixAny || ignorePreCheckPermission;</span>
<span class="source-line-no">3363</span><span id="line-3363"> }</span>
<span class="source-line-no">3364</span><span id="line-3364"></span>
<span class="source-line-no">3365</span><span id="line-3365"> public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) {</span>
<span class="source-line-no">3366</span><span id="line-3366"> this.ignorePreCheckPermission = ignorePreCheckPermission;</span>
<span class="source-line-no">3367</span><span id="line-3367"> }</span>
<span class="source-line-no">3368</span><span id="line-3368"></span>
<span class="source-line-no">3369</span><span id="line-3369"> /**</span>
<span class="source-line-no">3370</span><span id="line-3370"> * @param mm maximum number of regions to merge into a single region.</span>
<span class="source-line-no">3371</span><span id="line-3371"> */</span>
<span class="source-line-no">3372</span><span id="line-3372"> public void setMaxMerge(int mm) {</span>
<span class="source-line-no">3373</span><span id="line-3373"> this.maxMerge = mm;</span>
<span class="source-line-no">3374</span><span id="line-3374"> }</span>
<span class="source-line-no">3375</span><span id="line-3375"></span>
<span class="source-line-no">3376</span><span id="line-3376"> public int getMaxMerge() {</span>
<span class="source-line-no">3377</span><span id="line-3377"> return maxMerge;</span>
<span class="source-line-no">3378</span><span id="line-3378"> }</span>
<span class="source-line-no">3379</span><span id="line-3379"></span>
<span class="source-line-no">3380</span><span id="line-3380"> public void setMaxOverlapsToSideline(int mo) {</span>
<span class="source-line-no">3381</span><span id="line-3381"> this.maxOverlapsToSideline = mo;</span>
<span class="source-line-no">3382</span><span id="line-3382"> }</span>
<span class="source-line-no">3383</span><span id="line-3383"></span>
<span class="source-line-no">3384</span><span id="line-3384"> public int getMaxOverlapsToSideline() {</span>
<span class="source-line-no">3385</span><span id="line-3385"> return maxOverlapsToSideline;</span>
<span class="source-line-no">3386</span><span id="line-3386"> }</span>
<span class="source-line-no">3387</span><span id="line-3387"></span>
<span class="source-line-no">3388</span><span id="line-3388"> /**</span>
<span class="source-line-no">3389</span><span id="line-3389"> * Only check/fix tables specified by the list, Empty list means all tables are included.</span>
<span class="source-line-no">3390</span><span id="line-3390"> */</span>
<span class="source-line-no">3391</span><span id="line-3391"> boolean isTableIncluded(TableName table) {</span>
<span class="source-line-no">3392</span><span id="line-3392"> return (tablesIncluded.isEmpty()) || tablesIncluded.contains(table);</span>
<span class="source-line-no">3393</span><span id="line-3393"> }</span>
<span class="source-line-no">3394</span><span id="line-3394"></span>
<span class="source-line-no">3395</span><span id="line-3395"> public void includeTable(TableName table) {</span>
<span class="source-line-no">3396</span><span id="line-3396"> tablesIncluded.add(table);</span>
<span class="source-line-no">3397</span><span id="line-3397"> }</span>
<span class="source-line-no">3398</span><span id="line-3398"></span>
<span class="source-line-no">3399</span><span id="line-3399"> Set&lt;TableName&gt; getIncludedTables() {</span>
<span class="source-line-no">3400</span><span id="line-3400"> return new HashSet&lt;&gt;(tablesIncluded);</span>
<span class="source-line-no">3401</span><span id="line-3401"> }</span>
<span class="source-line-no">3402</span><span id="line-3402"></span>
<span class="source-line-no">3403</span><span id="line-3403"> /**</span>
<span class="source-line-no">3404</span><span id="line-3404"> * We are interested in only those tables that have not changed their state in hbase:meta during</span>
<span class="source-line-no">3405</span><span id="line-3405"> * the last few seconds specified by hbase.admin.fsck.timelag</span>
<span class="source-line-no">3406</span><span id="line-3406"> * @param seconds - the time in seconds</span>
<span class="source-line-no">3407</span><span id="line-3407"> */</span>
<span class="source-line-no">3408</span><span id="line-3408"> public void setTimeLag(long seconds) {</span>
<span class="source-line-no">3409</span><span id="line-3409"> timelag = seconds * 1000; // convert to milliseconds</span>
<span class="source-line-no">3410</span><span id="line-3410"> }</span>
<span class="source-line-no">3411</span><span id="line-3411"></span>
<span class="source-line-no">3412</span><span id="line-3412"> /**</span>
<span class="source-line-no">3413</span><span id="line-3413"> * @param sidelineDir - HDFS path to sideline data</span>
<span class="source-line-no">3414</span><span id="line-3414"> */</span>
<span class="source-line-no">3415</span><span id="line-3415"> public void setSidelineDir(String sidelineDir) {</span>
<span class="source-line-no">3416</span><span id="line-3416"> this.sidelineDir = new Path(sidelineDir);</span>
<span class="source-line-no">3417</span><span id="line-3417"> }</span>
<span class="source-line-no">3418</span><span id="line-3418"></span>
<span class="source-line-no">3419</span><span id="line-3419"> protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)</span>
<span class="source-line-no">3420</span><span id="line-3420"> throws IOException {</span>
<span class="source-line-no">3421</span><span id="line-3421"> return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);</span>
<span class="source-line-no">3422</span><span id="line-3422"> }</span>
<span class="source-line-no">3423</span><span id="line-3423"></span>
<span class="source-line-no">3424</span><span id="line-3424"> public HFileCorruptionChecker getHFilecorruptionChecker() {</span>
<span class="source-line-no">3425</span><span id="line-3425"> return hfcc;</span>
<span class="source-line-no">3426</span><span id="line-3426"> }</span>
<span class="source-line-no">3427</span><span id="line-3427"></span>
<span class="source-line-no">3428</span><span id="line-3428"> public void setHFileCorruptionChecker(HFileCorruptionChecker hfcc) {</span>
<span class="source-line-no">3429</span><span id="line-3429"> this.hfcc = hfcc;</span>
<span class="source-line-no">3430</span><span id="line-3430"> }</span>
<span class="source-line-no">3431</span><span id="line-3431"></span>
<span class="source-line-no">3432</span><span id="line-3432"> public void setRetCode(int code) {</span>
<span class="source-line-no">3433</span><span id="line-3433"> this.retcode = code;</span>
<span class="source-line-no">3434</span><span id="line-3434"> }</span>
<span class="source-line-no">3435</span><span id="line-3435"></span>
<span class="source-line-no">3436</span><span id="line-3436"> public int getRetCode() {</span>
<span class="source-line-no">3437</span><span id="line-3437"> return retcode;</span>
<span class="source-line-no">3438</span><span id="line-3438"> }</span>
<span class="source-line-no">3439</span><span id="line-3439"></span>
<span class="source-line-no">3440</span><span id="line-3440"> protected HBaseFsck printUsageAndExit() {</span>
<span class="source-line-no">3441</span><span id="line-3441"> StringWriter sw = new StringWriter(2048);</span>
<span class="source-line-no">3442</span><span id="line-3442"> PrintWriter out = new PrintWriter(sw);</span>
<span class="source-line-no">3443</span><span id="line-3443"> out.println("");</span>
<span class="source-line-no">3444</span><span id="line-3444"> out.println("-----------------------------------------------------------------------");</span>
<span class="source-line-no">3445</span><span id="line-3445"> out.println("NOTE: As of HBase version 2.0, the hbck tool is significantly changed.");</span>
<span class="source-line-no">3446</span><span id="line-3446"> out.println("In general, all Read-Only options are supported and can be be used");</span>
<span class="source-line-no">3447</span><span id="line-3447"> out.println("safely. Most -fix/ -repair options are NOT supported. Please see usage");</span>
<span class="source-line-no">3448</span><span id="line-3448"> out.println("below for details on which options are not supported.");</span>
<span class="source-line-no">3449</span><span id="line-3449"> out.println("-----------------------------------------------------------------------");</span>
<span class="source-line-no">3450</span><span id="line-3450"> out.println("");</span>
<span class="source-line-no">3451</span><span id="line-3451"> out.println("Usage: fsck [opts] {only tables}");</span>
<span class="source-line-no">3452</span><span id="line-3452"> out.println(" where [opts] are:");</span>
<span class="source-line-no">3453</span><span id="line-3453"> out.println(" -help Display help options (this)");</span>
<span class="source-line-no">3454</span><span id="line-3454"> out.println(" -details Display full report of all regions.");</span>
<span class="source-line-no">3455</span><span id="line-3455"> out.println(" -timelag &lt;timeInSeconds&gt; Process only regions that "</span>
<span class="source-line-no">3456</span><span id="line-3456"> + " have not experienced any metadata updates in the last " + " &lt;timeInSeconds&gt; seconds.");</span>
<span class="source-line-no">3457</span><span id="line-3457"> out.println(" -sleepBeforeRerun &lt;timeInSeconds&gt; Sleep this many seconds"</span>
<span class="source-line-no">3458</span><span id="line-3458"> + " before checking if the fix worked if run with -fix");</span>
<span class="source-line-no">3459</span><span id="line-3459"> out.println(" -summary Print only summary of the tables and status.");</span>
<span class="source-line-no">3460</span><span id="line-3460"> out.println(" -metaonly Only check the state of the hbase:meta table.");</span>
<span class="source-line-no">3461</span><span id="line-3461"> out.println(" -sidelineDir &lt;hdfs://&gt; HDFS path to backup existing meta.");</span>
<span class="source-line-no">3462</span><span id="line-3462"> out.println(</span>
<span class="source-line-no">3463</span><span id="line-3463"> " -boundaries Verify that regions boundaries are the same between META and store files.");</span>
<span class="source-line-no">3464</span><span id="line-3464"> out.println(" -exclusive Abort if another hbck is exclusive or fixing.");</span>
<span class="source-line-no">3465</span><span id="line-3465"></span>
<span class="source-line-no">3466</span><span id="line-3466"> out.println("");</span>
<span class="source-line-no">3467</span><span id="line-3467"> out.println(" Datafile Repair options: (expert features, use with caution!)");</span>
<span class="source-line-no">3468</span><span id="line-3468"> out.println(</span>
<span class="source-line-no">3469</span><span id="line-3469"> " -checkCorruptHFiles Check all Hfiles by opening them to make sure they are valid");</span>
<span class="source-line-no">3470</span><span id="line-3470"> out.println(</span>
<span class="source-line-no">3471</span><span id="line-3471"> " -sidelineCorruptHFiles Quarantine corrupted HFiles. implies -checkCorruptHFiles");</span>
<span class="source-line-no">3472</span><span id="line-3472"></span>
<span class="source-line-no">3473</span><span id="line-3473"> out.println("");</span>
<span class="source-line-no">3474</span><span id="line-3474"> out.println(" Replication options");</span>
<span class="source-line-no">3475</span><span id="line-3475"> out.println(" -fixReplication Deletes replication queues for removed peers");</span>
<span class="source-line-no">3476</span><span id="line-3476"></span>
<span class="source-line-no">3477</span><span id="line-3477"> out.println("");</span>
<span class="source-line-no">3478</span><span id="line-3478"> out.println(</span>
<span class="source-line-no">3479</span><span id="line-3479"> " Metadata Repair options supported as of version 2.0: (expert features, use with caution!)");</span>
<span class="source-line-no">3480</span><span id="line-3480"> out.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs.");</span>
<span class="source-line-no">3481</span><span id="line-3481"> out.println(" -fixReferenceFiles Try to offline lingering reference store files");</span>
<span class="source-line-no">3482</span><span id="line-3482"> out.println(" -fixHFileLinks Try to offline lingering HFileLinks");</span>
<span class="source-line-no">3483</span><span id="line-3483"> out.println(" -noHdfsChecking Don't load/check region info from HDFS."</span>
<span class="source-line-no">3484</span><span id="line-3484"> + " Assumes hbase:meta region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");</span>
<span class="source-line-no">3485</span><span id="line-3485"> out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");</span>
<span class="source-line-no">3486</span><span id="line-3486"></span>
<span class="source-line-no">3487</span><span id="line-3487"> out.println("");</span>
<span class="source-line-no">3488</span><span id="line-3488"> out.println("NOTE: Following options are NOT supported as of HBase version 2.0+.");</span>
<span class="source-line-no">3489</span><span id="line-3489"> out.println("");</span>
<span class="source-line-no">3490</span><span id="line-3490"> out.println(" UNSUPPORTED Metadata Repair options: (expert features, use with caution!)");</span>
<span class="source-line-no">3491</span><span id="line-3491"> out.println(</span>
<span class="source-line-no">3492</span><span id="line-3492"> " -fix Try to fix region assignments. This is for backwards compatibility");</span>
<span class="source-line-no">3493</span><span id="line-3493"> out.println(" -fixAssignments Try to fix region assignments. Replaces the old -fix");</span>
<span class="source-line-no">3494</span><span id="line-3494"> out.println(</span>
<span class="source-line-no">3495</span><span id="line-3495"> " -fixMeta Try to fix meta problems. This assumes HDFS region info is good.");</span>
<span class="source-line-no">3496</span><span id="line-3496"> out.println(" -fixHdfsHoles Try to fix region holes in hdfs.");</span>
<span class="source-line-no">3497</span><span id="line-3497"> out.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs");</span>
<span class="source-line-no">3498</span><span id="line-3498"> out.println(</span>
<span class="source-line-no">3499</span><span id="line-3499"> " -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");</span>
<span class="source-line-no">3500</span><span id="line-3500"> out.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs.");</span>
<span class="source-line-no">3501</span><span id="line-3501"> out.println(</span>
<span class="source-line-no">3502</span><span id="line-3502"> " -maxMerge &lt;n&gt; When fixing region overlaps, allow at most &lt;n&gt; regions to merge. (n="</span>
<span class="source-line-no">3503</span><span id="line-3503"> + DEFAULT_MAX_MERGE + " by default)");</span>
<span class="source-line-no">3504</span><span id="line-3504"> out.println(</span>
<span class="source-line-no">3505</span><span id="line-3505"> " -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps");</span>
<span class="source-line-no">3506</span><span id="line-3506"> out.println(</span>
<span class="source-line-no">3507</span><span id="line-3507"> " -maxOverlapsToSideline &lt;n&gt; When fixing region overlaps, allow at most &lt;n&gt; regions to sideline per group. (n="</span>
<span class="source-line-no">3508</span><span id="line-3508"> + DEFAULT_OVERLAPS_TO_SIDELINE + " by default)");</span>
<span class="source-line-no">3509</span><span id="line-3509"> out.println(" -fixSplitParents Try to force offline split parents to be online.");</span>
<span class="source-line-no">3510</span><span id="line-3510"> out.println(</span>
<span class="source-line-no">3511</span><span id="line-3511"> " -removeParents Try to offline and sideline lingering parents and keep daughter regions.");</span>
<span class="source-line-no">3512</span><span id="line-3512"> out.println(" -fixEmptyMetaCells Try to fix hbase:meta entries not referencing any region"</span>
<span class="source-line-no">3513</span><span id="line-3513"> + " (empty REGIONINFO_QUALIFIER rows)");</span>
<span class="source-line-no">3514</span><span id="line-3514"></span>
<span class="source-line-no">3515</span><span id="line-3515"> out.println("");</span>
<span class="source-line-no">3516</span><span id="line-3516"> out.println(" UNSUPPORTED Metadata Repair shortcuts");</span>
<span class="source-line-no">3517</span><span id="line-3517"> out.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles "</span>
<span class="source-line-no">3518</span><span id="line-3518"> + "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles"</span>
<span class="source-line-no">3519</span><span id="line-3519"> + "-fixHFileLinks");</span>
<span class="source-line-no">3520</span><span id="line-3520"> out.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");</span>
<span class="source-line-no">3521</span><span id="line-3521"> out.println("");</span>
<span class="source-line-no">3522</span><span id="line-3522"> out.println(" Replication options");</span>
<span class="source-line-no">3523</span><span id="line-3523"> out.println(" -fixReplication Deletes replication queues for removed peers");</span>
<span class="source-line-no">3524</span><span id="line-3524"> out.println(" -cleanReplicationBarrier [tableName] clean the replication barriers "</span>
<span class="source-line-no">3525</span><span id="line-3525"> + "of a specified table, tableName is required");</span>
<span class="source-line-no">3526</span><span id="line-3526"> out.flush();</span>
<span class="source-line-no">3527</span><span id="line-3527"> errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());</span>
<span class="source-line-no">3528</span><span id="line-3528"></span>
<span class="source-line-no">3529</span><span id="line-3529"> setRetCode(-2);</span>
<span class="source-line-no">3530</span><span id="line-3530"> return this;</span>
<span class="source-line-no">3531</span><span id="line-3531"> }</span>
<span class="source-line-no">3532</span><span id="line-3532"></span>
<span class="source-line-no">3533</span><span id="line-3533"> /**</span>
<span class="source-line-no">3534</span><span id="line-3534"> * Main program</span>
<span class="source-line-no">3535</span><span id="line-3535"> */</span>
<span class="source-line-no">3536</span><span id="line-3536"> public static void main(String[] args) throws Exception {</span>
<span class="source-line-no">3537</span><span id="line-3537"> // create a fsck object</span>
<span class="source-line-no">3538</span><span id="line-3538"> Configuration conf = HBaseConfiguration.create();</span>
<span class="source-line-no">3539</span><span id="line-3539"> Path hbasedir = CommonFSUtils.getRootDir(conf);</span>
<span class="source-line-no">3540</span><span id="line-3540"> URI defaultFs = hbasedir.getFileSystem(conf).getUri();</span>
<span class="source-line-no">3541</span><span id="line-3541"> CommonFSUtils.setFsDefault(conf, new Path(defaultFs));</span>
<span class="source-line-no">3542</span><span id="line-3542"> int ret = ToolRunner.run(new HBaseFsckTool(conf), args);</span>
<span class="source-line-no">3543</span><span id="line-3543"> System.exit(ret);</span>
<span class="source-line-no">3544</span><span id="line-3544"> }</span>
<span class="source-line-no">3545</span><span id="line-3545"></span>
<span class="source-line-no">3546</span><span id="line-3546"> /**</span>
<span class="source-line-no">3547</span><span id="line-3547"> * This is a Tool wrapper that gathers -Dxxx=yyy configuration settings from the command line.</span>
<span class="source-line-no">3548</span><span id="line-3548"> */</span>
<span class="source-line-no">3549</span><span id="line-3549"> static class HBaseFsckTool extends Configured implements Tool {</span>
<span class="source-line-no">3550</span><span id="line-3550"> HBaseFsckTool(Configuration conf) {</span>
<span class="source-line-no">3551</span><span id="line-3551"> super(conf);</span>
<span class="source-line-no">3552</span><span id="line-3552"> }</span>
<span class="source-line-no">3553</span><span id="line-3553"></span>
<span class="source-line-no">3554</span><span id="line-3554"> @Override</span>
<span class="source-line-no">3555</span><span id="line-3555"> public int run(String[] args) throws Exception {</span>
<span class="source-line-no">3556</span><span id="line-3556"> HBaseFsck hbck = new HBaseFsck(getConf());</span>
<span class="source-line-no">3557</span><span id="line-3557"> hbck.exec(hbck.executor, args);</span>
<span class="source-line-no">3558</span><span id="line-3558"> hbck.close();</span>
<span class="source-line-no">3559</span><span id="line-3559"> return hbck.getRetCode();</span>
<span class="source-line-no">3560</span><span id="line-3560"> }</span>
<span class="source-line-no">3561</span><span id="line-3561"> }</span>
<span class="source-line-no">3562</span><span id="line-3562"></span>
<span class="source-line-no">3563</span><span id="line-3563"> public HBaseFsck exec(ExecutorService exec, String[] args)</span>
<span class="source-line-no">3564</span><span id="line-3564"> throws KeeperException, IOException, InterruptedException, ReplicationException {</span>
<span class="source-line-no">3565</span><span id="line-3565"> long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;</span>
<span class="source-line-no">3566</span><span id="line-3566"></span>
<span class="source-line-no">3567</span><span id="line-3567"> boolean checkCorruptHFiles = false;</span>
<span class="source-line-no">3568</span><span id="line-3568"> boolean sidelineCorruptHFiles = false;</span>
<span class="source-line-no">3569</span><span id="line-3569"></span>
<span class="source-line-no">3570</span><span id="line-3570"> // Process command-line args.</span>
<span class="source-line-no">3571</span><span id="line-3571"> for (int i = 0; i &lt; args.length; i++) {</span>
<span class="source-line-no">3572</span><span id="line-3572"> String cmd = args[i];</span>
<span class="source-line-no">3573</span><span id="line-3573"> if (cmd.equals("-help") || cmd.equals("-h")) {</span>
<span class="source-line-no">3574</span><span id="line-3574"> return printUsageAndExit();</span>
<span class="source-line-no">3575</span><span id="line-3575"> } else if (cmd.equals("-details")) {</span>
<span class="source-line-no">3576</span><span id="line-3576"> setDisplayFullReport();</span>
<span class="source-line-no">3577</span><span id="line-3577"> } else if (cmd.equals("-exclusive")) {</span>
<span class="source-line-no">3578</span><span id="line-3578"> setForceExclusive();</span>
<span class="source-line-no">3579</span><span id="line-3579"> } else if (cmd.equals("-timelag")) {</span>
<span class="source-line-no">3580</span><span id="line-3580"> if (i == args.length - 1) {</span>
<span class="source-line-no">3581</span><span id="line-3581"> errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");</span>
<span class="source-line-no">3582</span><span id="line-3582"> return printUsageAndExit();</span>
<span class="source-line-no">3583</span><span id="line-3583"> }</span>
<span class="source-line-no">3584</span><span id="line-3584"> try {</span>
<span class="source-line-no">3585</span><span id="line-3585"> long timelag = Long.parseLong(args[++i]);</span>
<span class="source-line-no">3586</span><span id="line-3586"> setTimeLag(timelag);</span>
<span class="source-line-no">3587</span><span id="line-3587"> } catch (NumberFormatException e) {</span>
<span class="source-line-no">3588</span><span id="line-3588"> errors.reportError(ERROR_CODE.WRONG_USAGE, "-timelag needs a numeric value.");</span>
<span class="source-line-no">3589</span><span id="line-3589"> return printUsageAndExit();</span>
<span class="source-line-no">3590</span><span id="line-3590"> }</span>
<span class="source-line-no">3591</span><span id="line-3591"> } else if (cmd.equals("-sleepBeforeRerun")) {</span>
<span class="source-line-no">3592</span><span id="line-3592"> if (i == args.length - 1) {</span>
<span class="source-line-no">3593</span><span id="line-3593"> errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sleepBeforeRerun needs a value.");</span>
<span class="source-line-no">3594</span><span id="line-3594"> return printUsageAndExit();</span>
<span class="source-line-no">3595</span><span id="line-3595"> }</span>
<span class="source-line-no">3596</span><span id="line-3596"> try {</span>
<span class="source-line-no">3597</span><span id="line-3597"> sleepBeforeRerun = Long.parseLong(args[++i]);</span>
<span class="source-line-no">3598</span><span id="line-3598"> } catch (NumberFormatException e) {</span>
<span class="source-line-no">3599</span><span id="line-3599"> errors.reportError(ERROR_CODE.WRONG_USAGE, "-sleepBeforeRerun needs a numeric value.");</span>
<span class="source-line-no">3600</span><span id="line-3600"> return printUsageAndExit();</span>
<span class="source-line-no">3601</span><span id="line-3601"> }</span>
<span class="source-line-no">3602</span><span id="line-3602"> } else if (cmd.equals("-sidelineDir")) {</span>
<span class="source-line-no">3603</span><span id="line-3603"> if (i == args.length - 1) {</span>
<span class="source-line-no">3604</span><span id="line-3604"> errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -sidelineDir needs a value.");</span>
<span class="source-line-no">3605</span><span id="line-3605"> return printUsageAndExit();</span>
<span class="source-line-no">3606</span><span id="line-3606"> }</span>
<span class="source-line-no">3607</span><span id="line-3607"> setSidelineDir(args[++i]);</span>
<span class="source-line-no">3608</span><span id="line-3608"> } else if (cmd.equals("-fix")) {</span>
<span class="source-line-no">3609</span><span id="line-3609"> errors.reportError(ERROR_CODE.WRONG_USAGE,</span>
<span class="source-line-no">3610</span><span id="line-3610"> "This option is deprecated, please use -fixAssignments instead.");</span>
<span class="source-line-no">3611</span><span id="line-3611"> setFixAssignments(true);</span>
<span class="source-line-no">3612</span><span id="line-3612"> } else if (cmd.equals("-fixAssignments")) {</span>
<span class="source-line-no">3613</span><span id="line-3613"> setFixAssignments(true);</span>
<span class="source-line-no">3614</span><span id="line-3614"> } else if (cmd.equals("-fixMeta")) {</span>
<span class="source-line-no">3615</span><span id="line-3615"> setFixMeta(true);</span>
<span class="source-line-no">3616</span><span id="line-3616"> } else if (cmd.equals("-noHdfsChecking")) {</span>
<span class="source-line-no">3617</span><span id="line-3617"> setCheckHdfs(false);</span>
<span class="source-line-no">3618</span><span id="line-3618"> } else if (cmd.equals("-fixHdfsHoles")) {</span>
<span class="source-line-no">3619</span><span id="line-3619"> setFixHdfsHoles(true);</span>
<span class="source-line-no">3620</span><span id="line-3620"> } else if (cmd.equals("-fixHdfsOrphans")) {</span>
<span class="source-line-no">3621</span><span id="line-3621"> setFixHdfsOrphans(true);</span>
<span class="source-line-no">3622</span><span id="line-3622"> } else if (cmd.equals("-fixTableOrphans")) {</span>
<span class="source-line-no">3623</span><span id="line-3623"> setFixTableOrphans(true);</span>
<span class="source-line-no">3624</span><span id="line-3624"> } else if (cmd.equals("-fixHdfsOverlaps")) {</span>
<span class="source-line-no">3625</span><span id="line-3625"> setFixHdfsOverlaps(true);</span>
<span class="source-line-no">3626</span><span id="line-3626"> } else if (cmd.equals("-fixVersionFile")) {</span>
<span class="source-line-no">3627</span><span id="line-3627"> setFixVersionFile(true);</span>
<span class="source-line-no">3628</span><span id="line-3628"> } else if (cmd.equals("-sidelineBigOverlaps")) {</span>
<span class="source-line-no">3629</span><span id="line-3629"> setSidelineBigOverlaps(true);</span>
<span class="source-line-no">3630</span><span id="line-3630"> } else if (cmd.equals("-fixSplitParents")) {</span>
<span class="source-line-no">3631</span><span id="line-3631"> setFixSplitParents(true);</span>
<span class="source-line-no">3632</span><span id="line-3632"> } else if (cmd.equals("-removeParents")) {</span>
<span class="source-line-no">3633</span><span id="line-3633"> setRemoveParents(true);</span>
<span class="source-line-no">3634</span><span id="line-3634"> } else if (cmd.equals("-ignorePreCheckPermission")) {</span>
<span class="source-line-no">3635</span><span id="line-3635"> setIgnorePreCheckPermission(true);</span>
<span class="source-line-no">3636</span><span id="line-3636"> } else if (cmd.equals("-checkCorruptHFiles")) {</span>
<span class="source-line-no">3637</span><span id="line-3637"> checkCorruptHFiles = true;</span>
<span class="source-line-no">3638</span><span id="line-3638"> } else if (cmd.equals("-sidelineCorruptHFiles")) {</span>
<span class="source-line-no">3639</span><span id="line-3639"> sidelineCorruptHFiles = true;</span>
<span class="source-line-no">3640</span><span id="line-3640"> } else if (cmd.equals("-fixReferenceFiles")) {</span>
<span class="source-line-no">3641</span><span id="line-3641"> setFixReferenceFiles(true);</span>
<span class="source-line-no">3642</span><span id="line-3642"> } else if (cmd.equals("-fixHFileLinks")) {</span>
<span class="source-line-no">3643</span><span id="line-3643"> setFixHFileLinks(true);</span>
<span class="source-line-no">3644</span><span id="line-3644"> } else if (cmd.equals("-fixEmptyMetaCells")) {</span>
<span class="source-line-no">3645</span><span id="line-3645"> setFixEmptyMetaCells(true);</span>
<span class="source-line-no">3646</span><span id="line-3646"> } else if (cmd.equals("-repair")) {</span>
<span class="source-line-no">3647</span><span id="line-3647"> // this attempts to merge overlapping hdfs regions, needs testing</span>
<span class="source-line-no">3648</span><span id="line-3648"> // under load</span>
<span class="source-line-no">3649</span><span id="line-3649"> setFixHdfsHoles(true);</span>
<span class="source-line-no">3650</span><span id="line-3650"> setFixHdfsOrphans(true);</span>
<span class="source-line-no">3651</span><span id="line-3651"> setFixMeta(true);</span>
<span class="source-line-no">3652</span><span id="line-3652"> setFixAssignments(true);</span>
<span class="source-line-no">3653</span><span id="line-3653"> setFixHdfsOverlaps(true);</span>
<span class="source-line-no">3654</span><span id="line-3654"> setFixVersionFile(true);</span>
<span class="source-line-no">3655</span><span id="line-3655"> setSidelineBigOverlaps(true);</span>
<span class="source-line-no">3656</span><span id="line-3656"> setFixSplitParents(false);</span>
<span class="source-line-no">3657</span><span id="line-3657"> setCheckHdfs(true);</span>
<span class="source-line-no">3658</span><span id="line-3658"> setFixReferenceFiles(true);</span>
<span class="source-line-no">3659</span><span id="line-3659"> setFixHFileLinks(true);</span>
<span class="source-line-no">3660</span><span id="line-3660"> } else if (cmd.equals("-repairHoles")) {</span>
<span class="source-line-no">3661</span><span id="line-3661"> // this will make all missing hdfs regions available but may lose data</span>
<span class="source-line-no">3662</span><span id="line-3662"> setFixHdfsHoles(true);</span>
<span class="source-line-no">3663</span><span id="line-3663"> setFixHdfsOrphans(false);</span>
<span class="source-line-no">3664</span><span id="line-3664"> setFixMeta(true);</span>
<span class="source-line-no">3665</span><span id="line-3665"> setFixAssignments(true);</span>
<span class="source-line-no">3666</span><span id="line-3666"> setFixHdfsOverlaps(false);</span>
<span class="source-line-no">3667</span><span id="line-3667"> setSidelineBigOverlaps(false);</span>
<span class="source-line-no">3668</span><span id="line-3668"> setFixSplitParents(false);</span>
<span class="source-line-no">3669</span><span id="line-3669"> setCheckHdfs(true);</span>
<span class="source-line-no">3670</span><span id="line-3670"> } else if (cmd.equals("-maxOverlapsToSideline")) {</span>
<span class="source-line-no">3671</span><span id="line-3671"> if (i == args.length - 1) {</span>
<span class="source-line-no">3672</span><span id="line-3672"> errors.reportError(ERROR_CODE.WRONG_USAGE,</span>
<span class="source-line-no">3673</span><span id="line-3673"> "-maxOverlapsToSideline needs a numeric value argument.");</span>
<span class="source-line-no">3674</span><span id="line-3674"> return printUsageAndExit();</span>
<span class="source-line-no">3675</span><span id="line-3675"> }</span>
<span class="source-line-no">3676</span><span id="line-3676"> try {</span>
<span class="source-line-no">3677</span><span id="line-3677"> int maxOverlapsToSideline = Integer.parseInt(args[++i]);</span>
<span class="source-line-no">3678</span><span id="line-3678"> setMaxOverlapsToSideline(maxOverlapsToSideline);</span>
<span class="source-line-no">3679</span><span id="line-3679"> } catch (NumberFormatException e) {</span>
<span class="source-line-no">3680</span><span id="line-3680"> errors.reportError(ERROR_CODE.WRONG_USAGE,</span>
<span class="source-line-no">3681</span><span id="line-3681"> "-maxOverlapsToSideline needs a numeric value argument.");</span>
<span class="source-line-no">3682</span><span id="line-3682"> return printUsageAndExit();</span>
<span class="source-line-no">3683</span><span id="line-3683"> }</span>
<span class="source-line-no">3684</span><span id="line-3684"> } else if (cmd.equals("-maxMerge")) {</span>
<span class="source-line-no">3685</span><span id="line-3685"> if (i == args.length - 1) {</span>
<span class="source-line-no">3686</span><span id="line-3686"> errors.reportError(ERROR_CODE.WRONG_USAGE, "-maxMerge needs a numeric value argument.");</span>
<span class="source-line-no">3687</span><span id="line-3687"> return printUsageAndExit();</span>
<span class="source-line-no">3688</span><span id="line-3688"> }</span>
<span class="source-line-no">3689</span><span id="line-3689"> try {</span>
<span class="source-line-no">3690</span><span id="line-3690"> int maxMerge = Integer.parseInt(args[++i]);</span>
<span class="source-line-no">3691</span><span id="line-3691"> setMaxMerge(maxMerge);</span>
<span class="source-line-no">3692</span><span id="line-3692"> } catch (NumberFormatException e) {</span>
<span class="source-line-no">3693</span><span id="line-3693"> errors.reportError(ERROR_CODE.WRONG_USAGE, "-maxMerge needs a numeric value argument.");</span>
<span class="source-line-no">3694</span><span id="line-3694"> return printUsageAndExit();</span>
<span class="source-line-no">3695</span><span id="line-3695"> }</span>
<span class="source-line-no">3696</span><span id="line-3696"> } else if (cmd.equals("-summary")) {</span>
<span class="source-line-no">3697</span><span id="line-3697"> setSummary();</span>
<span class="source-line-no">3698</span><span id="line-3698"> } else if (cmd.equals("-metaonly")) {</span>
<span class="source-line-no">3699</span><span id="line-3699"> setCheckMetaOnly();</span>
<span class="source-line-no">3700</span><span id="line-3700"> } else if (cmd.equals("-boundaries")) {</span>
<span class="source-line-no">3701</span><span id="line-3701"> setRegionBoundariesCheck();</span>
<span class="source-line-no">3702</span><span id="line-3702"> } else if (cmd.equals("-fixReplication")) {</span>
<span class="source-line-no">3703</span><span id="line-3703"> setFixReplication(true);</span>
<span class="source-line-no">3704</span><span id="line-3704"> } else if (cmd.equals("-cleanReplicationBarrier")) {</span>
<span class="source-line-no">3705</span><span id="line-3705"> setCleanReplicationBarrier(true);</span>
<span class="source-line-no">3706</span><span id="line-3706"> if (args[++i].startsWith("-")) {</span>
<span class="source-line-no">3707</span><span id="line-3707"> printUsageAndExit();</span>
<span class="source-line-no">3708</span><span id="line-3708"> }</span>
<span class="source-line-no">3709</span><span id="line-3709"> setCleanReplicationBarrierTable(args[i]);</span>
<span class="source-line-no">3710</span><span id="line-3710"> } else if (cmd.startsWith("-")) {</span>
<span class="source-line-no">3711</span><span id="line-3711"> errors.reportError(ERROR_CODE.WRONG_USAGE, "Unrecognized option:" + cmd);</span>
<span class="source-line-no">3712</span><span id="line-3712"> return printUsageAndExit();</span>
<span class="source-line-no">3713</span><span id="line-3713"> } else {</span>
<span class="source-line-no">3714</span><span id="line-3714"> includeTable(TableName.valueOf(cmd));</span>
<span class="source-line-no">3715</span><span id="line-3715"> errors.print("Allow checking/fixes for table: " + cmd);</span>
<span class="source-line-no">3716</span><span id="line-3716"> }</span>
<span class="source-line-no">3717</span><span id="line-3717"> }</span>
<span class="source-line-no">3718</span><span id="line-3718"></span>
<span class="source-line-no">3719</span><span id="line-3719"> errors.print("HBaseFsck command line options: " + StringUtils.join(args, " "));</span>
<span class="source-line-no">3720</span><span id="line-3720"></span>
<span class="source-line-no">3721</span><span id="line-3721"> // pre-check current user has FS write permission or not</span>
<span class="source-line-no">3722</span><span id="line-3722"> try {</span>
<span class="source-line-no">3723</span><span id="line-3723"> preCheckPermission();</span>
<span class="source-line-no">3724</span><span id="line-3724"> } catch (IOException ioe) {</span>
<span class="source-line-no">3725</span><span id="line-3725"> Runtime.getRuntime().exit(-1);</span>
<span class="source-line-no">3726</span><span id="line-3726"> }</span>
<span class="source-line-no">3727</span><span id="line-3727"></span>
<span class="source-line-no">3728</span><span id="line-3728"> // do the real work of hbck</span>
<span class="source-line-no">3729</span><span id="line-3729"> connect();</span>
<span class="source-line-no">3730</span><span id="line-3730"></span>
<span class="source-line-no">3731</span><span id="line-3731"> // after connecting to server above, we have server version</span>
<span class="source-line-no">3732</span><span id="line-3732"> // check if unsupported option is specified based on server version</span>
<span class="source-line-no">3733</span><span id="line-3733"> if (!isOptionsSupported(args)) {</span>
<span class="source-line-no">3734</span><span id="line-3734"> return printUsageAndExit();</span>
<span class="source-line-no">3735</span><span id="line-3735"> }</span>
<span class="source-line-no">3736</span><span id="line-3736"></span>
<span class="source-line-no">3737</span><span id="line-3737"> try {</span>
<span class="source-line-no">3738</span><span id="line-3738"> // if corrupt file mode is on, first fix them since they may be opened later</span>
<span class="source-line-no">3739</span><span id="line-3739"> if (checkCorruptHFiles || sidelineCorruptHFiles) {</span>
<span class="source-line-no">3740</span><span id="line-3740"> LOG.info("Checking all hfiles for corruption");</span>
<span class="source-line-no">3741</span><span id="line-3741"> HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);</span>
<span class="source-line-no">3742</span><span id="line-3742"> setHFileCorruptionChecker(hfcc); // so we can get result</span>
<span class="source-line-no">3743</span><span id="line-3743"> Collection&lt;TableName&gt; tables = getIncludedTables();</span>
<span class="source-line-no">3744</span><span id="line-3744"> Collection&lt;Path&gt; tableDirs = new ArrayList&lt;&gt;();</span>
<span class="source-line-no">3745</span><span id="line-3745"> Path rootdir = CommonFSUtils.getRootDir(getConf());</span>
<span class="source-line-no">3746</span><span id="line-3746"> if (tables.size() &gt; 0) {</span>
<span class="source-line-no">3747</span><span id="line-3747"> for (TableName t : tables) {</span>
<span class="source-line-no">3748</span><span id="line-3748"> tableDirs.add(CommonFSUtils.getTableDir(rootdir, t));</span>
<span class="source-line-no">3749</span><span id="line-3749"> }</span>
<span class="source-line-no">3750</span><span id="line-3750"> } else {</span>
<span class="source-line-no">3751</span><span id="line-3751"> tableDirs = FSUtils.getTableDirs(CommonFSUtils.getCurrentFileSystem(getConf()), rootdir);</span>
<span class="source-line-no">3752</span><span id="line-3752"> }</span>
<span class="source-line-no">3753</span><span id="line-3753"> hfcc.checkTables(tableDirs);</span>
<span class="source-line-no">3754</span><span id="line-3754"> hfcc.report(errors);</span>
<span class="source-line-no">3755</span><span id="line-3755"> }</span>
<span class="source-line-no">3756</span><span id="line-3756"></span>
<span class="source-line-no">3757</span><span id="line-3757"> // check and fix table integrity, region consistency.</span>
<span class="source-line-no">3758</span><span id="line-3758"> int code = onlineHbck();</span>
<span class="source-line-no">3759</span><span id="line-3759"> setRetCode(code);</span>
<span class="source-line-no">3760</span><span id="line-3760"> // If we have changed the HBase state it is better to run hbck again</span>
<span class="source-line-no">3761</span><span id="line-3761"> // to see if we haven't broken something else in the process.</span>
<span class="source-line-no">3762</span><span id="line-3762"> // We run it only once more because otherwise we can easily fall into</span>
<span class="source-line-no">3763</span><span id="line-3763"> // an infinite loop.</span>
<span class="source-line-no">3764</span><span id="line-3764"> if (shouldRerun()) {</span>
<span class="source-line-no">3765</span><span id="line-3765"> try {</span>
<span class="source-line-no">3766</span><span id="line-3766"> LOG.info("Sleeping " + sleepBeforeRerun + "ms before re-checking after fix...");</span>
<span class="source-line-no">3767</span><span id="line-3767"> Thread.sleep(sleepBeforeRerun);</span>
<span class="source-line-no">3768</span><span id="line-3768"> } catch (InterruptedException ie) {</span>
<span class="source-line-no">3769</span><span id="line-3769"> LOG.warn("Interrupted while sleeping");</span>
<span class="source-line-no">3770</span><span id="line-3770"> return this;</span>
<span class="source-line-no">3771</span><span id="line-3771"> }</span>
<span class="source-line-no">3772</span><span id="line-3772"> // Just report</span>
<span class="source-line-no">3773</span><span id="line-3773"> setFixAssignments(false);</span>
<span class="source-line-no">3774</span><span id="line-3774"> setFixMeta(false);</span>
<span class="source-line-no">3775</span><span id="line-3775"> setFixHdfsHoles(false);</span>
<span class="source-line-no">3776</span><span id="line-3776"> setFixHdfsOverlaps(false);</span>
<span class="source-line-no">3777</span><span id="line-3777"> setFixVersionFile(false);</span>
<span class="source-line-no">3778</span><span id="line-3778"> setFixTableOrphans(false);</span>
<span class="source-line-no">3779</span><span id="line-3779"> errors.resetErrors();</span>
<span class="source-line-no">3780</span><span id="line-3780"> code = onlineHbck();</span>
<span class="source-line-no">3781</span><span id="line-3781"> setRetCode(code);</span>
<span class="source-line-no">3782</span><span id="line-3782"> }</span>
<span class="source-line-no">3783</span><span id="line-3783"> } finally {</span>
<span class="source-line-no">3784</span><span id="line-3784"> IOUtils.closeQuietly(this, e -&gt; LOG.warn("", e));</span>
<span class="source-line-no">3785</span><span id="line-3785"> }</span>
<span class="source-line-no">3786</span><span id="line-3786"> return this;</span>
<span class="source-line-no">3787</span><span id="line-3787"> }</span>
<span class="source-line-no">3788</span><span id="line-3788"></span>
<span class="source-line-no">3789</span><span id="line-3789"> private boolean isOptionsSupported(String[] args) {</span>
<span class="source-line-no">3790</span><span id="line-3790"> boolean result = true;</span>
<span class="source-line-no">3791</span><span id="line-3791"> String hbaseServerVersion = status.getHBaseVersion();</span>
<span class="source-line-no">3792</span><span id="line-3792"> if (VersionInfo.compareVersion("2.any.any", hbaseServerVersion) &lt; 0) {</span>
<span class="source-line-no">3793</span><span id="line-3793"> // Process command-line args.</span>
<span class="source-line-no">3794</span><span id="line-3794"> for (String arg : args) {</span>
<span class="source-line-no">3795</span><span id="line-3795"> if (unsupportedOptionsInV2.contains(arg)) {</span>
<span class="source-line-no">3796</span><span id="line-3796"> errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION,</span>
<span class="source-line-no">3797</span><span id="line-3797"> "option '" + arg + "' is not " + "supported!");</span>
<span class="source-line-no">3798</span><span id="line-3798"> result = false;</span>
<span class="source-line-no">3799</span><span id="line-3799"> break;</span>
<span class="source-line-no">3800</span><span id="line-3800"> }</span>
<span class="source-line-no">3801</span><span id="line-3801"> }</span>
<span class="source-line-no">3802</span><span id="line-3802"> }</span>
<span class="source-line-no">3803</span><span id="line-3803"> return result;</span>
<span class="source-line-no">3804</span><span id="line-3804"> }</span>
<span class="source-line-no">3805</span><span id="line-3805"></span>
<span class="source-line-no">3806</span><span id="line-3806"> public void setCleanReplicationBarrierTable(String cleanReplicationBarrierTable) {</span>
<span class="source-line-no">3807</span><span id="line-3807"> this.cleanReplicationBarrierTable = TableName.valueOf(cleanReplicationBarrierTable);</span>
<span class="source-line-no">3808</span><span id="line-3808"> }</span>
<span class="source-line-no">3809</span><span id="line-3809"></span>
<span class="source-line-no">3810</span><span id="line-3810"> public void cleanReplicationBarrier() throws IOException {</span>
<span class="source-line-no">3811</span><span id="line-3811"> if (!cleanReplicationBarrier || cleanReplicationBarrierTable == null) {</span>
<span class="source-line-no">3812</span><span id="line-3812"> return;</span>
<span class="source-line-no">3813</span><span id="line-3813"> }</span>
<span class="source-line-no">3814</span><span id="line-3814"> if (cleanReplicationBarrierTable.isSystemTable()) {</span>
<span class="source-line-no">3815</span><span id="line-3815"> errors.reportError(ERROR_CODE.INVALID_TABLE,</span>
<span class="source-line-no">3816</span><span id="line-3816"> "invalid table: " + cleanReplicationBarrierTable);</span>
<span class="source-line-no">3817</span><span id="line-3817"> return;</span>
<span class="source-line-no">3818</span><span id="line-3818"> }</span>
<span class="source-line-no">3819</span><span id="line-3819"></span>
<span class="source-line-no">3820</span><span id="line-3820"> boolean isGlobalScope = false;</span>
<span class="source-line-no">3821</span><span id="line-3821"> try {</span>
<span class="source-line-no">3822</span><span id="line-3822"> isGlobalScope = admin.getDescriptor(cleanReplicationBarrierTable).hasGlobalReplicationScope();</span>
<span class="source-line-no">3823</span><span id="line-3823"> } catch (TableNotFoundException e) {</span>
<span class="source-line-no">3824</span><span id="line-3824"> LOG.info("we may need to clean some erroneous data due to bugs");</span>
<span class="source-line-no">3825</span><span id="line-3825"> }</span>
<span class="source-line-no">3826</span><span id="line-3826"></span>
<span class="source-line-no">3827</span><span id="line-3827"> if (isGlobalScope) {</span>
<span class="source-line-no">3828</span><span id="line-3828"> errors.reportError(ERROR_CODE.INVALID_TABLE,</span>
<span class="source-line-no">3829</span><span id="line-3829"> "table's replication scope is global: " + cleanReplicationBarrierTable);</span>
<span class="source-line-no">3830</span><span id="line-3830"> return;</span>
<span class="source-line-no">3831</span><span id="line-3831"> }</span>
<span class="source-line-no">3832</span><span id="line-3832"> List&lt;byte[]&gt; regionNames = new ArrayList&lt;&gt;();</span>
<span class="source-line-no">3833</span><span id="line-3833"> Scan barrierScan = new Scan();</span>
<span class="source-line-no">3834</span><span id="line-3834"> barrierScan.setCaching(100);</span>
<span class="source-line-no">3835</span><span id="line-3835"> barrierScan.addFamily(HConstants.REPLICATION_BARRIER_FAMILY);</span>
<span class="source-line-no">3836</span><span id="line-3836"> barrierScan</span>
<span class="source-line-no">3837</span><span id="line-3837"> .withStartRow(ClientMetaTableAccessor.getTableStartRowForMeta(cleanReplicationBarrierTable,</span>
<span class="source-line-no">3838</span><span id="line-3838"> ClientMetaTableAccessor.QueryType.REGION))</span>
<span class="source-line-no">3839</span><span id="line-3839"> .withStopRow(ClientMetaTableAccessor.getTableStopRowForMeta(cleanReplicationBarrierTable,</span>
<span class="source-line-no">3840</span><span id="line-3840"> ClientMetaTableAccessor.QueryType.REGION));</span>
<span class="source-line-no">3841</span><span id="line-3841"> Result result;</span>
<span class="source-line-no">3842</span><span id="line-3842"> try (ResultScanner scanner = meta.getScanner(barrierScan)) {</span>
<span class="source-line-no">3843</span><span id="line-3843"> while ((result = scanner.next()) != null) {</span>
<span class="source-line-no">3844</span><span id="line-3844"> regionNames.add(result.getRow());</span>
<span class="source-line-no">3845</span><span id="line-3845"> }</span>
<span class="source-line-no">3846</span><span id="line-3846"> }</span>
<span class="source-line-no">3847</span><span id="line-3847"> if (regionNames.size() &lt;= 0) {</span>
<span class="source-line-no">3848</span><span id="line-3848"> errors.reportError(ERROR_CODE.INVALID_TABLE,</span>
<span class="source-line-no">3849</span><span id="line-3849"> "there is no barriers of this table: " + cleanReplicationBarrierTable);</span>
<span class="source-line-no">3850</span><span id="line-3850"> return;</span>
<span class="source-line-no">3851</span><span id="line-3851"> }</span>
<span class="source-line-no">3852</span><span id="line-3852"> ReplicationQueueStorage queueStorage =</span>
<span class="source-line-no">3853</span><span id="line-3853"> ReplicationStorageFactory.getReplicationQueueStorage(connection, getConf());</span>
<span class="source-line-no">3854</span><span id="line-3854"> List&lt;ReplicationPeerDescription&gt; peerDescriptions = admin.listReplicationPeers();</span>
<span class="source-line-no">3855</span><span id="line-3855"> if (peerDescriptions != null &amp;&amp; peerDescriptions.size() &gt; 0) {</span>
<span class="source-line-no">3856</span><span id="line-3856"> List&lt;String&gt; peers = peerDescriptions.stream()</span>
<span class="source-line-no">3857</span><span id="line-3857"> .filter(</span>
<span class="source-line-no">3858</span><span id="line-3858"> peerConfig -&gt; peerConfig.getPeerConfig().needToReplicate(cleanReplicationBarrierTable))</span>
<span class="source-line-no">3859</span><span id="line-3859"> .map(peerConfig -&gt; peerConfig.getPeerId()).collect(Collectors.toList());</span>
<span class="source-line-no">3860</span><span id="line-3860"> try {</span>
<span class="source-line-no">3861</span><span id="line-3861"> List&lt;String&gt; batch = new ArrayList&lt;&gt;();</span>
<span class="source-line-no">3862</span><span id="line-3862"> for (String peer : peers) {</span>
<span class="source-line-no">3863</span><span id="line-3863"> for (byte[] regionName : regionNames) {</span>
<span class="source-line-no">3864</span><span id="line-3864"> batch.add(RegionInfo.encodeRegionName(regionName));</span>
<span class="source-line-no">3865</span><span id="line-3865"> if (batch.size() % 100 == 0) {</span>
<span class="source-line-no">3866</span><span id="line-3866"> queueStorage.removeLastSequenceIds(peer, batch);</span>
<span class="source-line-no">3867</span><span id="line-3867"> batch.clear();</span>
<span class="source-line-no">3868</span><span id="line-3868"> }</span>
<span class="source-line-no">3869</span><span id="line-3869"> }</span>
<span class="source-line-no">3870</span><span id="line-3870"> if (batch.size() &gt; 0) {</span>
<span class="source-line-no">3871</span><span id="line-3871"> queueStorage.removeLastSequenceIds(peer, batch);</span>
<span class="source-line-no">3872</span><span id="line-3872"> batch.clear();</span>
<span class="source-line-no">3873</span><span id="line-3873"> }</span>
<span class="source-line-no">3874</span><span id="line-3874"> }</span>
<span class="source-line-no">3875</span><span id="line-3875"> } catch (ReplicationException re) {</span>
<span class="source-line-no">3876</span><span id="line-3876"> throw new IOException(re);</span>
<span class="source-line-no">3877</span><span id="line-3877"> }</span>
<span class="source-line-no">3878</span><span id="line-3878"> }</span>
<span class="source-line-no">3879</span><span id="line-3879"> for (byte[] regionName : regionNames) {</span>
<span class="source-line-no">3880</span><span id="line-3880"> meta.delete(new Delete(regionName).addFamily(HConstants.REPLICATION_BARRIER_FAMILY));</span>
<span class="source-line-no">3881</span><span id="line-3881"> }</span>
<span class="source-line-no">3882</span><span id="line-3882"> setShouldRerun();</span>
<span class="source-line-no">3883</span><span id="line-3883"> }</span>
<span class="source-line-no">3884</span><span id="line-3884"></span>
<span class="source-line-no">3885</span><span id="line-3885"> /**</span>
<span class="source-line-no">3886</span><span id="line-3886"> * ls -r for debugging purposes</span>
<span class="source-line-no">3887</span><span id="line-3887"> */</span>
<span class="source-line-no">3888</span><span id="line-3888"> void debugLsr(Path p) throws IOException {</span>
<span class="source-line-no">3889</span><span id="line-3889"> debugLsr(getConf(), p, errors);</span>
<span class="source-line-no">3890</span><span id="line-3890"> }</span>
<span class="source-line-no">3891</span><span id="line-3891"></span>
<span class="source-line-no">3892</span><span id="line-3892"> /**</span>
<span class="source-line-no">3893</span><span id="line-3893"> * ls -r for debugging purposes</span>
<span class="source-line-no">3894</span><span id="line-3894"> */</span>
<span class="source-line-no">3895</span><span id="line-3895"> public static void debugLsr(Configuration conf, Path p) throws IOException {</span>
<span class="source-line-no">3896</span><span id="line-3896"> debugLsr(conf, p, new PrintingErrorReporter());</span>
<span class="source-line-no">3897</span><span id="line-3897"> }</span>
<span class="source-line-no">3898</span><span id="line-3898"></span>
<span class="source-line-no">3899</span><span id="line-3899"> /**</span>
<span class="source-line-no">3900</span><span id="line-3900"> * ls -r for debugging purposes</span>
<span class="source-line-no">3901</span><span id="line-3901"> */</span>
<span class="source-line-no">3902</span><span id="line-3902"> public static void debugLsr(Configuration conf, Path p, HbckErrorReporter errors)</span>
<span class="source-line-no">3903</span><span id="line-3903"> throws IOException {</span>
<span class="source-line-no">3904</span><span id="line-3904"> if (!LOG.isDebugEnabled() || p == null) {</span>
<span class="source-line-no">3905</span><span id="line-3905"> return;</span>
<span class="source-line-no">3906</span><span id="line-3906"> }</span>
<span class="source-line-no">3907</span><span id="line-3907"> FileSystem fs = p.getFileSystem(conf);</span>
<span class="source-line-no">3908</span><span id="line-3908"></span>
<span class="source-line-no">3909</span><span id="line-3909"> if (!fs.exists(p)) {</span>
<span class="source-line-no">3910</span><span id="line-3910"> // nothing</span>
<span class="source-line-no">3911</span><span id="line-3911"> return;</span>
<span class="source-line-no">3912</span><span id="line-3912"> }</span>
<span class="source-line-no">3913</span><span id="line-3913"> errors.print(p.toString());</span>
<span class="source-line-no">3914</span><span id="line-3914"></span>
<span class="source-line-no">3915</span><span id="line-3915"> if (fs.isFile(p)) {</span>
<span class="source-line-no">3916</span><span id="line-3916"> return;</span>
<span class="source-line-no">3917</span><span id="line-3917"> }</span>
<span class="source-line-no">3918</span><span id="line-3918"></span>
<span class="source-line-no">3919</span><span id="line-3919"> if (fs.getFileStatus(p).isDirectory()) {</span>
<span class="source-line-no">3920</span><span id="line-3920"> FileStatus[] fss = fs.listStatus(p);</span>
<span class="source-line-no">3921</span><span id="line-3921"> for (FileStatus status : fss) {</span>
<span class="source-line-no">3922</span><span id="line-3922"> debugLsr(conf, status.getPath(), errors);</span>
<span class="source-line-no">3923</span><span id="line-3923"> }</span>
<span class="source-line-no">3924</span><span id="line-3924"> }</span>
<span class="source-line-no">3925</span><span id="line-3925"> }</span>
<span class="source-line-no">3926</span><span id="line-3926">}</span>
</pre>
</div>
</main>
</body>
</html>