| <!DOCTYPE HTML> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (17) --> |
| <title>Source code</title> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="source: package: org.apache.hadoop.hbase.util, class: FSUtils, interface: ProgressReporter"> |
| <meta name="generator" content="javadoc/SourceToHTMLConverter"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| </head> |
| <body class="source-page"> |
| <main role="main"> |
| <div class="source-container"> |
| <pre><span class="source-line-no">001</span><span id="line-1">/*</span> |
| <span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span> |
| <span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span> |
| <span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span> |
| <span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span> |
| <span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span> |
| <span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span> |
| <span class="source-line-no">009</span><span id="line-9"> *</span> |
| <span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="source-line-no">011</span><span id="line-11"> *</span> |
| <span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span> |
| <span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span> |
| <span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> |
| <span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span> |
| <span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span> |
| <span class="source-line-no">017</span><span id="line-17"> */</span> |
| <span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.util;</span> |
| <span class="source-line-no">019</span><span id="line-19"></span> |
| <span class="source-line-no">020</span><span id="line-20">import static org.apache.hadoop.hbase.util.LocatedBlockHelper.getLocatedBlockLocations;</span> |
| <span class="source-line-no">021</span><span id="line-21">import static org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction.SAFEMODE_GET;</span> |
| <span class="source-line-no">022</span><span id="line-22"></span> |
| <span class="source-line-no">023</span><span id="line-23">import edu.umd.cs.findbugs.annotations.CheckForNull;</span> |
| <span class="source-line-no">024</span><span id="line-24">import java.io.ByteArrayInputStream;</span> |
| <span class="source-line-no">025</span><span id="line-25">import java.io.DataInputStream;</span> |
| <span class="source-line-no">026</span><span id="line-26">import java.io.EOFException;</span> |
| <span class="source-line-no">027</span><span id="line-27">import java.io.FileNotFoundException;</span> |
| <span class="source-line-no">028</span><span id="line-28">import java.io.IOException;</span> |
| <span class="source-line-no">029</span><span id="line-29">import java.io.InterruptedIOException;</span> |
| <span class="source-line-no">030</span><span id="line-30">import java.lang.reflect.InvocationTargetException;</span> |
| <span class="source-line-no">031</span><span id="line-31">import java.lang.reflect.Method;</span> |
| <span class="source-line-no">032</span><span id="line-32">import java.net.InetSocketAddress;</span> |
| <span class="source-line-no">033</span><span id="line-33">import java.net.URI;</span> |
| <span class="source-line-no">034</span><span id="line-34">import java.util.ArrayList;</span> |
| <span class="source-line-no">035</span><span id="line-35">import java.util.Arrays;</span> |
| <span class="source-line-no">036</span><span id="line-36">import java.util.Collection;</span> |
| <span class="source-line-no">037</span><span id="line-37">import java.util.Collections;</span> |
| <span class="source-line-no">038</span><span id="line-38">import java.util.HashMap;</span> |
| <span class="source-line-no">039</span><span id="line-39">import java.util.HashSet;</span> |
| <span class="source-line-no">040</span><span id="line-40">import java.util.Iterator;</span> |
| <span class="source-line-no">041</span><span id="line-41">import java.util.List;</span> |
| <span class="source-line-no">042</span><span id="line-42">import java.util.Locale;</span> |
| <span class="source-line-no">043</span><span id="line-43">import java.util.Map;</span> |
| <span class="source-line-no">044</span><span id="line-44">import java.util.Optional;</span> |
| <span class="source-line-no">045</span><span id="line-45">import java.util.Set;</span> |
| <span class="source-line-no">046</span><span id="line-46">import java.util.Vector;</span> |
| <span class="source-line-no">047</span><span id="line-47">import java.util.concurrent.ConcurrentHashMap;</span> |
| <span class="source-line-no">048</span><span id="line-48">import java.util.concurrent.ExecutionException;</span> |
| <span class="source-line-no">049</span><span id="line-49">import java.util.concurrent.ExecutorService;</span> |
| <span class="source-line-no">050</span><span id="line-50">import java.util.concurrent.Executors;</span> |
| <span class="source-line-no">051</span><span id="line-51">import java.util.concurrent.Future;</span> |
| <span class="source-line-no">052</span><span id="line-52">import java.util.concurrent.FutureTask;</span> |
| <span class="source-line-no">053</span><span id="line-53">import java.util.concurrent.ThreadPoolExecutor;</span> |
| <span class="source-line-no">054</span><span id="line-54">import java.util.concurrent.TimeUnit;</span> |
| <span class="source-line-no">055</span><span id="line-55">import java.util.regex.Pattern;</span> |
| <span class="source-line-no">056</span><span id="line-56">import org.apache.commons.lang3.ArrayUtils;</span> |
| <span class="source-line-no">057</span><span id="line-57">import org.apache.hadoop.conf.Configuration;</span> |
| <span class="source-line-no">058</span><span id="line-58">import org.apache.hadoop.fs.BlockLocation;</span> |
| <span class="source-line-no">059</span><span id="line-59">import org.apache.hadoop.fs.FSDataInputStream;</span> |
| <span class="source-line-no">060</span><span id="line-60">import org.apache.hadoop.fs.FSDataOutputStream;</span> |
| <span class="source-line-no">061</span><span id="line-61">import org.apache.hadoop.fs.FileStatus;</span> |
| <span class="source-line-no">062</span><span id="line-62">import org.apache.hadoop.fs.FileSystem;</span> |
| <span class="source-line-no">063</span><span id="line-63">import org.apache.hadoop.fs.FileUtil;</span> |
| <span class="source-line-no">064</span><span id="line-64">import org.apache.hadoop.fs.Path;</span> |
| <span class="source-line-no">065</span><span id="line-65">import org.apache.hadoop.fs.PathFilter;</span> |
| <span class="source-line-no">066</span><span id="line-66">import org.apache.hadoop.fs.StorageType;</span> |
| <span class="source-line-no">067</span><span id="line-67">import org.apache.hadoop.fs.permission.FsPermission;</span> |
| <span class="source-line-no">068</span><span id="line-68">import org.apache.hadoop.hbase.ClusterId;</span> |
| <span class="source-line-no">069</span><span id="line-69">import org.apache.hadoop.hbase.HConstants;</span> |
| <span class="source-line-no">070</span><span id="line-70">import org.apache.hadoop.hbase.HDFSBlocksDistribution;</span> |
| <span class="source-line-no">071</span><span id="line-71">import org.apache.hadoop.hbase.TableName;</span> |
| <span class="source-line-no">072</span><span id="line-72">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span> |
| <span class="source-line-no">073</span><span id="line-73">import org.apache.hadoop.hbase.client.RegionInfo;</span> |
| <span class="source-line-no">074</span><span id="line-74">import org.apache.hadoop.hbase.client.RegionInfoBuilder;</span> |
| <span class="source-line-no">075</span><span id="line-75">import org.apache.hadoop.hbase.exceptions.DeserializationException;</span> |
| <span class="source-line-no">076</span><span id="line-76">import org.apache.hadoop.hbase.fs.HFileSystem;</span> |
| <span class="source-line-no">077</span><span id="line-77">import org.apache.hadoop.hbase.io.HFileLink;</span> |
| <span class="source-line-no">078</span><span id="line-78">import org.apache.hadoop.hbase.master.HMaster;</span> |
| <span class="source-line-no">079</span><span id="line-79">import org.apache.hadoop.hbase.regionserver.StoreFileInfo;</span> |
| <span class="source-line-no">080</span><span id="line-80">import org.apache.hadoop.hdfs.DFSClient;</span> |
| <span class="source-line-no">081</span><span id="line-81">import org.apache.hadoop.hdfs.DFSHedgedReadMetrics;</span> |
| <span class="source-line-no">082</span><span id="line-82">import org.apache.hadoop.hdfs.DFSUtil;</span> |
| <span class="source-line-no">083</span><span id="line-83">import org.apache.hadoop.hdfs.DistributedFileSystem;</span> |
| <span class="source-line-no">084</span><span id="line-84">import org.apache.hadoop.hdfs.client.HdfsDataInputStream;</span> |
| <span class="source-line-no">085</span><span id="line-85">import org.apache.hadoop.hdfs.protocol.DatanodeInfo;</span> |
| <span class="source-line-no">086</span><span id="line-86">import org.apache.hadoop.hdfs.protocol.LocatedBlock;</span> |
| <span class="source-line-no">087</span><span id="line-87">import org.apache.hadoop.io.IOUtils;</span> |
| <span class="source-line-no">088</span><span id="line-88">import org.apache.hadoop.ipc.RemoteException;</span> |
| <span class="source-line-no">089</span><span id="line-89">import org.apache.yetus.audience.InterfaceAudience;</span> |
| <span class="source-line-no">090</span><span id="line-90">import org.slf4j.Logger;</span> |
| <span class="source-line-no">091</span><span id="line-91">import org.slf4j.LoggerFactory;</span> |
| <span class="source-line-no">092</span><span id="line-92"></span> |
| <span class="source-line-no">093</span><span id="line-93">import org.apache.hbase.thirdparty.com.google.common.base.Throwables;</span> |
| <span class="source-line-no">094</span><span id="line-94">import org.apache.hbase.thirdparty.com.google.common.collect.Iterators;</span> |
| <span class="source-line-no">095</span><span id="line-95">import org.apache.hbase.thirdparty.com.google.common.collect.Sets;</span> |
| <span class="source-line-no">096</span><span id="line-96">import org.apache.hbase.thirdparty.com.google.common.primitives.Ints;</span> |
| <span class="source-line-no">097</span><span id="line-97">import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;</span> |
| <span class="source-line-no">098</span><span id="line-98"></span> |
| <span class="source-line-no">099</span><span id="line-99">import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;</span> |
| <span class="source-line-no">100</span><span id="line-100">import org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos;</span> |
| <span class="source-line-no">101</span><span id="line-101"></span> |
| <span class="source-line-no">102</span><span id="line-102">/**</span> |
| <span class="source-line-no">103</span><span id="line-103"> * Utility methods for interacting with the underlying file system.</span> |
| <span class="source-line-no">104</span><span id="line-104"> */</span> |
| <span class="source-line-no">105</span><span id="line-105">@InterfaceAudience.Private</span> |
| <span class="source-line-no">106</span><span id="line-106">public final class FSUtils {</span> |
| <span class="source-line-no">107</span><span id="line-107"> private static final Logger LOG = LoggerFactory.getLogger(FSUtils.class);</span> |
| <span class="source-line-no">108</span><span id="line-108"></span> |
| <span class="source-line-no">109</span><span id="line-109"> private static final String THREAD_POOLSIZE = "hbase.client.localityCheck.threadPoolSize";</span> |
| <span class="source-line-no">110</span><span id="line-110"> private static final int DEFAULT_THREAD_POOLSIZE = 2;</span> |
| <span class="source-line-no">111</span><span id="line-111"></span> |
| <span class="source-line-no">112</span><span id="line-112"> /** Set to true on Windows platforms */</span> |
| <span class="source-line-no">113</span><span id="line-113"> // currently only used in testing. TODO refactor into a test class</span> |
| <span class="source-line-no">114</span><span id="line-114"> public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");</span> |
| <span class="source-line-no">115</span><span id="line-115"></span> |
| <span class="source-line-no">116</span><span id="line-116"> private FSUtils() {</span> |
| <span class="source-line-no">117</span><span id="line-117"> }</span> |
| <span class="source-line-no">118</span><span id="line-118"></span> |
| <span class="source-line-no">119</span><span id="line-119"> /** Returns True is <code>fs</code> is instance of DistributedFileSystem n */</span> |
| <span class="source-line-no">120</span><span id="line-120"> public static boolean isDistributedFileSystem(final FileSystem fs) throws IOException {</span> |
| <span class="source-line-no">121</span><span id="line-121"> FileSystem fileSystem = fs;</span> |
| <span class="source-line-no">122</span><span id="line-122"> // If passed an instance of HFileSystem, it fails instanceof DistributedFileSystem.</span> |
| <span class="source-line-no">123</span><span id="line-123"> // Check its backing fs for dfs-ness.</span> |
| <span class="source-line-no">124</span><span id="line-124"> if (fs instanceof HFileSystem) {</span> |
| <span class="source-line-no">125</span><span id="line-125"> fileSystem = ((HFileSystem) fs).getBackingFs();</span> |
| <span class="source-line-no">126</span><span id="line-126"> }</span> |
| <span class="source-line-no">127</span><span id="line-127"> return fileSystem instanceof DistributedFileSystem;</span> |
| <span class="source-line-no">128</span><span id="line-128"> }</span> |
| <span class="source-line-no">129</span><span id="line-129"></span> |
| <span class="source-line-no">130</span><span id="line-130"> /**</span> |
| <span class="source-line-no">131</span><span id="line-131"> * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the</span> |
| <span class="source-line-no">132</span><span id="line-132"> * '/a/b/c' part. If you passed in 'hdfs://a/b/c and b/c, it would return true. Does not consider</span> |
| <span class="source-line-no">133</span><span id="line-133"> * schema; i.e. if schemas different but path or subpath matches, the two will equate.</span> |
| <span class="source-line-no">134</span><span id="line-134"> * @param pathToSearch Path we will be trying to match.</span> |
| <span class="source-line-no">135</span><span id="line-135"> * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code></span> |
| <span class="source-line-no">136</span><span id="line-136"> */</span> |
| <span class="source-line-no">137</span><span id="line-137"> public static boolean isMatchingTail(final Path pathToSearch, final Path pathTail) {</span> |
| <span class="source-line-no">138</span><span id="line-138"> Path tailPath = pathTail;</span> |
| <span class="source-line-no">139</span><span id="line-139"> String tailName;</span> |
| <span class="source-line-no">140</span><span id="line-140"> Path toSearch = pathToSearch;</span> |
| <span class="source-line-no">141</span><span id="line-141"> String toSearchName;</span> |
| <span class="source-line-no">142</span><span id="line-142"> boolean result = false;</span> |
| <span class="source-line-no">143</span><span id="line-143"></span> |
| <span class="source-line-no">144</span><span id="line-144"> if (pathToSearch.depth() != pathTail.depth()) {</span> |
| <span class="source-line-no">145</span><span id="line-145"> return false;</span> |
| <span class="source-line-no">146</span><span id="line-146"> }</span> |
| <span class="source-line-no">147</span><span id="line-147"></span> |
| <span class="source-line-no">148</span><span id="line-148"> do {</span> |
| <span class="source-line-no">149</span><span id="line-149"> tailName = tailPath.getName();</span> |
| <span class="source-line-no">150</span><span id="line-150"> if (tailName == null || tailName.isEmpty()) {</span> |
| <span class="source-line-no">151</span><span id="line-151"> result = true;</span> |
| <span class="source-line-no">152</span><span id="line-152"> break;</span> |
| <span class="source-line-no">153</span><span id="line-153"> }</span> |
| <span class="source-line-no">154</span><span id="line-154"> toSearchName = toSearch.getName();</span> |
| <span class="source-line-no">155</span><span id="line-155"> if (toSearchName == null || toSearchName.isEmpty()) {</span> |
| <span class="source-line-no">156</span><span id="line-156"> break;</span> |
| <span class="source-line-no">157</span><span id="line-157"> }</span> |
| <span class="source-line-no">158</span><span id="line-158"> // Move up a parent on each path for next go around. Path doesn't let us go off the end.</span> |
| <span class="source-line-no">159</span><span id="line-159"> tailPath = tailPath.getParent();</span> |
| <span class="source-line-no">160</span><span id="line-160"> toSearch = toSearch.getParent();</span> |
| <span class="source-line-no">161</span><span id="line-161"> } while (tailName.equals(toSearchName));</span> |
| <span class="source-line-no">162</span><span id="line-162"> return result;</span> |
| <span class="source-line-no">163</span><span id="line-163"> }</span> |
| <span class="source-line-no">164</span><span id="line-164"></span> |
| <span class="source-line-no">165</span><span id="line-165"> /**</span> |
| <span class="source-line-no">166</span><span id="line-166"> * Delete the region directory if exists.</span> |
| <span class="source-line-no">167</span><span id="line-167"> * @return True if deleted the region directory.</span> |
| <span class="source-line-no">168</span><span id="line-168"> */</span> |
| <span class="source-line-no">169</span><span id="line-169"> public static boolean deleteRegionDir(final Configuration conf, final RegionInfo hri)</span> |
| <span class="source-line-no">170</span><span id="line-170"> throws IOException {</span> |
| <span class="source-line-no">171</span><span id="line-171"> Path rootDir = CommonFSUtils.getRootDir(conf);</span> |
| <span class="source-line-no">172</span><span id="line-172"> FileSystem fs = rootDir.getFileSystem(conf);</span> |
| <span class="source-line-no">173</span><span id="line-173"> return CommonFSUtils.deleteDirectory(fs,</span> |
| <span class="source-line-no">174</span><span id="line-174"> new Path(CommonFSUtils.getTableDir(rootDir, hri.getTable()), hri.getEncodedName()));</span> |
| <span class="source-line-no">175</span><span id="line-175"> }</span> |
| <span class="source-line-no">176</span><span id="line-176"></span> |
| <span class="source-line-no">177</span><span id="line-177"> /**</span> |
| <span class="source-line-no">178</span><span id="line-178"> * Create the specified file on the filesystem. By default, this will:</span> |
| <span class="source-line-no">179</span><span id="line-179"> * <ol></span> |
| <span class="source-line-no">180</span><span id="line-180"> * <li>overwrite the file if it exists</li></span> |
| <span class="source-line-no">181</span><span id="line-181"> * <li>apply the umask in the configuration (if it is enabled)</li></span> |
| <span class="source-line-no">182</span><span id="line-182"> * <li>use the fs configured buffer size (or 4096 if not set)</li></span> |
| <span class="source-line-no">183</span><span id="line-183"> * <li>use the configured column family replication or default replication if</span> |
| <span class="source-line-no">184</span><span id="line-184"> * {@link ColumnFamilyDescriptorBuilder#DEFAULT_DFS_REPLICATION}</li></span> |
| <span class="source-line-no">185</span><span id="line-185"> * <li>use the default block size</li></span> |
| <span class="source-line-no">186</span><span id="line-186"> * <li>not track progress</li></span> |
| <span class="source-line-no">187</span><span id="line-187"> * </ol></span> |
| <span class="source-line-no">188</span><span id="line-188"> * @param conf configurations</span> |
| <span class="source-line-no">189</span><span id="line-189"> * @param fs {@link FileSystem} on which to write the file</span> |
| <span class="source-line-no">190</span><span id="line-190"> * @param path {@link Path} to the file to write</span> |
| <span class="source-line-no">191</span><span id="line-191"> * @param perm permissions</span> |
| <span class="source-line-no">192</span><span id="line-192"> * @param favoredNodes favored data nodes</span> |
| <span class="source-line-no">193</span><span id="line-193"> * @return output stream to the created file</span> |
| <span class="source-line-no">194</span><span id="line-194"> * @throws IOException if the file cannot be created</span> |
| <span class="source-line-no">195</span><span id="line-195"> */</span> |
| <span class="source-line-no">196</span><span id="line-196"> public static FSDataOutputStream create(Configuration conf, FileSystem fs, Path path,</span> |
| <span class="source-line-no">197</span><span id="line-197"> FsPermission perm, InetSocketAddress[] favoredNodes) throws IOException {</span> |
| <span class="source-line-no">198</span><span id="line-198"> if (fs instanceof HFileSystem) {</span> |
| <span class="source-line-no">199</span><span id="line-199"> FileSystem backingFs = ((HFileSystem) fs).getBackingFs();</span> |
| <span class="source-line-no">200</span><span id="line-200"> if (backingFs instanceof DistributedFileSystem) {</span> |
| <span class="source-line-no">201</span><span id="line-201"> short replication = Short.parseShort(conf.get(ColumnFamilyDescriptorBuilder.DFS_REPLICATION,</span> |
| <span class="source-line-no">202</span><span id="line-202"> String.valueOf(ColumnFamilyDescriptorBuilder.DEFAULT_DFS_REPLICATION)));</span> |
| <span class="source-line-no">203</span><span id="line-203"> DistributedFileSystem.HdfsDataOutputStreamBuilder builder =</span> |
| <span class="source-line-no">204</span><span id="line-204"> ((DistributedFileSystem) backingFs).createFile(path).recursive().permission(perm)</span> |
| <span class="source-line-no">205</span><span id="line-205"> .create();</span> |
| <span class="source-line-no">206</span><span id="line-206"> if (favoredNodes != null) {</span> |
| <span class="source-line-no">207</span><span id="line-207"> builder.favoredNodes(favoredNodes);</span> |
| <span class="source-line-no">208</span><span id="line-208"> }</span> |
| <span class="source-line-no">209</span><span id="line-209"> if (replication > 0) {</span> |
| <span class="source-line-no">210</span><span id="line-210"> builder.replication(replication);</span> |
| <span class="source-line-no">211</span><span id="line-211"> }</span> |
| <span class="source-line-no">212</span><span id="line-212"> return builder.build();</span> |
| <span class="source-line-no">213</span><span id="line-213"> }</span> |
| <span class="source-line-no">214</span><span id="line-214"></span> |
| <span class="source-line-no">215</span><span id="line-215"> }</span> |
| <span class="source-line-no">216</span><span id="line-216"> return CommonFSUtils.create(fs, path, perm, true);</span> |
| <span class="source-line-no">217</span><span id="line-217"> }</span> |
| <span class="source-line-no">218</span><span id="line-218"></span> |
| <span class="source-line-no">219</span><span id="line-219"> /**</span> |
| <span class="source-line-no">220</span><span id="line-220"> * Checks to see if the specified file system is available</span> |
| <span class="source-line-no">221</span><span id="line-221"> * @param fs filesystem</span> |
| <span class="source-line-no">222</span><span id="line-222"> * @throws IOException e</span> |
| <span class="source-line-no">223</span><span id="line-223"> */</span> |
| <span class="source-line-no">224</span><span id="line-224"> public static void checkFileSystemAvailable(final FileSystem fs) throws IOException {</span> |
| <span class="source-line-no">225</span><span id="line-225"> if (!(fs instanceof DistributedFileSystem)) {</span> |
| <span class="source-line-no">226</span><span id="line-226"> return;</span> |
| <span class="source-line-no">227</span><span id="line-227"> }</span> |
| <span class="source-line-no">228</span><span id="line-228"> IOException exception = null;</span> |
| <span class="source-line-no">229</span><span id="line-229"> DistributedFileSystem dfs = (DistributedFileSystem) fs;</span> |
| <span class="source-line-no">230</span><span id="line-230"> try {</span> |
| <span class="source-line-no">231</span><span id="line-231"> if (dfs.exists(new Path("/"))) {</span> |
| <span class="source-line-no">232</span><span id="line-232"> return;</span> |
| <span class="source-line-no">233</span><span id="line-233"> }</span> |
| <span class="source-line-no">234</span><span id="line-234"> } catch (IOException e) {</span> |
| <span class="source-line-no">235</span><span id="line-235"> exception = e instanceof RemoteException ? ((RemoteException) e).unwrapRemoteException() : e;</span> |
| <span class="source-line-no">236</span><span id="line-236"> }</span> |
| <span class="source-line-no">237</span><span id="line-237"> try {</span> |
| <span class="source-line-no">238</span><span id="line-238"> fs.close();</span> |
| <span class="source-line-no">239</span><span id="line-239"> } catch (Exception e) {</span> |
| <span class="source-line-no">240</span><span id="line-240"> LOG.error("file system close failed: ", e);</span> |
| <span class="source-line-no">241</span><span id="line-241"> }</span> |
| <span class="source-line-no">242</span><span id="line-242"> throw new IOException("File system is not available", exception);</span> |
| <span class="source-line-no">243</span><span id="line-243"> }</span> |
| <span class="source-line-no">244</span><span id="line-244"></span> |
| <span class="source-line-no">245</span><span id="line-245"> /**</span> |
| <span class="source-line-no">246</span><span id="line-246"> * Inquire the Active NameNode's safe mode status.</span> |
| <span class="source-line-no">247</span><span id="line-247"> * @param dfs A DistributedFileSystem object representing the underlying HDFS.</span> |
| <span class="source-line-no">248</span><span id="line-248"> * @return whether we're in safe mode</span> |
| <span class="source-line-no">249</span><span id="line-249"> */</span> |
| <span class="source-line-no">250</span><span id="line-250"> private static boolean isInSafeMode(DistributedFileSystem dfs) throws IOException {</span> |
| <span class="source-line-no">251</span><span id="line-251"> return dfs.setSafeMode(SAFEMODE_GET, true);</span> |
| <span class="source-line-no">252</span><span id="line-252"> }</span> |
| <span class="source-line-no">253</span><span id="line-253"></span> |
| <span class="source-line-no">254</span><span id="line-254"> /**</span> |
| <span class="source-line-no">255</span><span id="line-255"> * Check whether dfs is in safemode.</span> |
| <span class="source-line-no">256</span><span id="line-256"> */</span> |
| <span class="source-line-no">257</span><span id="line-257"> public static void checkDfsSafeMode(final Configuration conf) throws IOException {</span> |
| <span class="source-line-no">258</span><span id="line-258"> boolean isInSafeMode = false;</span> |
| <span class="source-line-no">259</span><span id="line-259"> FileSystem fs = FileSystem.get(conf);</span> |
| <span class="source-line-no">260</span><span id="line-260"> if (fs instanceof DistributedFileSystem) {</span> |
| <span class="source-line-no">261</span><span id="line-261"> DistributedFileSystem dfs = (DistributedFileSystem) fs;</span> |
| <span class="source-line-no">262</span><span id="line-262"> isInSafeMode = isInSafeMode(dfs);</span> |
| <span class="source-line-no">263</span><span id="line-263"> }</span> |
| <span class="source-line-no">264</span><span id="line-264"> if (isInSafeMode) {</span> |
| <span class="source-line-no">265</span><span id="line-265"> throw new IOException("File system is in safemode, it can't be written now");</span> |
| <span class="source-line-no">266</span><span id="line-266"> }</span> |
| <span class="source-line-no">267</span><span id="line-267"> }</span> |
| <span class="source-line-no">268</span><span id="line-268"></span> |
| <span class="source-line-no">269</span><span id="line-269"> /**</span> |
| <span class="source-line-no">270</span><span id="line-270"> * Verifies current version of file system</span> |
| <span class="source-line-no">271</span><span id="line-271"> * @param fs filesystem object</span> |
| <span class="source-line-no">272</span><span id="line-272"> * @param rootdir root hbase directory</span> |
| <span class="source-line-no">273</span><span id="line-273"> * @return null if no version file exists, version string otherwise</span> |
| <span class="source-line-no">274</span><span id="line-274"> * @throws IOException if the version file fails to open</span> |
| <span class="source-line-no">275</span><span id="line-275"> * @throws DeserializationException if the version data cannot be translated into a version</span> |
| <span class="source-line-no">276</span><span id="line-276"> */</span> |
| <span class="source-line-no">277</span><span id="line-277"> public static String getVersion(FileSystem fs, Path rootdir)</span> |
| <span class="source-line-no">278</span><span id="line-278"> throws IOException, DeserializationException {</span> |
| <span class="source-line-no">279</span><span id="line-279"> final Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);</span> |
| <span class="source-line-no">280</span><span id="line-280"> FileStatus[] status = null;</span> |
| <span class="source-line-no">281</span><span id="line-281"> try {</span> |
| <span class="source-line-no">282</span><span id="line-282"> // hadoop 2.0 throws FNFE if directory does not exist.</span> |
| <span class="source-line-no">283</span><span id="line-283"> // hadoop 1.0 returns null if directory does not exist.</span> |
| <span class="source-line-no">284</span><span id="line-284"> status = fs.listStatus(versionFile);</span> |
| <span class="source-line-no">285</span><span id="line-285"> } catch (FileNotFoundException fnfe) {</span> |
| <span class="source-line-no">286</span><span id="line-286"> return null;</span> |
| <span class="source-line-no">287</span><span id="line-287"> }</span> |
| <span class="source-line-no">288</span><span id="line-288"> if (ArrayUtils.getLength(status) == 0) {</span> |
| <span class="source-line-no">289</span><span id="line-289"> return null;</span> |
| <span class="source-line-no">290</span><span id="line-290"> }</span> |
| <span class="source-line-no">291</span><span id="line-291"> String version = null;</span> |
| <span class="source-line-no">292</span><span id="line-292"> byte[] content = new byte[(int) status[0].getLen()];</span> |
| <span class="source-line-no">293</span><span id="line-293"> FSDataInputStream s = fs.open(versionFile);</span> |
| <span class="source-line-no">294</span><span id="line-294"> try {</span> |
| <span class="source-line-no">295</span><span id="line-295"> IOUtils.readFully(s, content, 0, content.length);</span> |
| <span class="source-line-no">296</span><span id="line-296"> if (ProtobufUtil.isPBMagicPrefix(content)) {</span> |
| <span class="source-line-no">297</span><span id="line-297"> version = parseVersionFrom(content);</span> |
| <span class="source-line-no">298</span><span id="line-298"> } else {</span> |
| <span class="source-line-no">299</span><span id="line-299"> // Presume it pre-pb format.</span> |
| <span class="source-line-no">300</span><span id="line-300"> try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(content))) {</span> |
| <span class="source-line-no">301</span><span id="line-301"> version = dis.readUTF();</span> |
| <span class="source-line-no">302</span><span id="line-302"> }</span> |
| <span class="source-line-no">303</span><span id="line-303"> }</span> |
| <span class="source-line-no">304</span><span id="line-304"> } catch (EOFException eof) {</span> |
| <span class="source-line-no">305</span><span id="line-305"> LOG.warn("Version file was empty, odd, will try to set it.");</span> |
| <span class="source-line-no">306</span><span id="line-306"> } finally {</span> |
| <span class="source-line-no">307</span><span id="line-307"> s.close();</span> |
| <span class="source-line-no">308</span><span id="line-308"> }</span> |
| <span class="source-line-no">309</span><span id="line-309"> return version;</span> |
| <span class="source-line-no">310</span><span id="line-310"> }</span> |
| <span class="source-line-no">311</span><span id="line-311"></span> |
| <span class="source-line-no">312</span><span id="line-312"> /**</span> |
| <span class="source-line-no">313</span><span id="line-313"> * Parse the content of the ${HBASE_ROOTDIR}/hbase.version file.</span> |
| <span class="source-line-no">314</span><span id="line-314"> * @param bytes The byte content of the hbase.version file</span> |
| <span class="source-line-no">315</span><span id="line-315"> * @return The version found in the file as a String</span> |
| <span class="source-line-no">316</span><span id="line-316"> * @throws DeserializationException if the version data cannot be translated into a version</span> |
| <span class="source-line-no">317</span><span id="line-317"> */</span> |
| <span class="source-line-no">318</span><span id="line-318"> static String parseVersionFrom(final byte[] bytes) throws DeserializationException {</span> |
| <span class="source-line-no">319</span><span id="line-319"> ProtobufUtil.expectPBMagicPrefix(bytes);</span> |
| <span class="source-line-no">320</span><span id="line-320"> int pblen = ProtobufUtil.lengthOfPBMagic();</span> |
| <span class="source-line-no">321</span><span id="line-321"> FSProtos.HBaseVersionFileContent.Builder builder =</span> |
| <span class="source-line-no">322</span><span id="line-322"> FSProtos.HBaseVersionFileContent.newBuilder();</span> |
| <span class="source-line-no">323</span><span id="line-323"> try {</span> |
| <span class="source-line-no">324</span><span id="line-324"> ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);</span> |
| <span class="source-line-no">325</span><span id="line-325"> return builder.getVersion();</span> |
| <span class="source-line-no">326</span><span id="line-326"> } catch (IOException e) {</span> |
| <span class="source-line-no">327</span><span id="line-327"> // Convert</span> |
| <span class="source-line-no">328</span><span id="line-328"> throw new DeserializationException(e);</span> |
| <span class="source-line-no">329</span><span id="line-329"> }</span> |
| <span class="source-line-no">330</span><span id="line-330"> }</span> |
| <span class="source-line-no">331</span><span id="line-331"></span> |
| <span class="source-line-no">332</span><span id="line-332"> /**</span> |
| <span class="source-line-no">333</span><span id="line-333"> * Create the content to write into the ${HBASE_ROOTDIR}/hbase.version file.</span> |
| <span class="source-line-no">334</span><span id="line-334"> * @param version Version to persist</span> |
| <span class="source-line-no">335</span><span id="line-335"> * @return Serialized protobuf with <code>version</code> content and a bit of pb magic for a</span> |
| <span class="source-line-no">336</span><span id="line-336"> * prefix.</span> |
| <span class="source-line-no">337</span><span id="line-337"> */</span> |
| <span class="source-line-no">338</span><span id="line-338"> static byte[] toVersionByteArray(final String version) {</span> |
| <span class="source-line-no">339</span><span id="line-339"> FSProtos.HBaseVersionFileContent.Builder builder =</span> |
| <span class="source-line-no">340</span><span id="line-340"> FSProtos.HBaseVersionFileContent.newBuilder();</span> |
| <span class="source-line-no">341</span><span id="line-341"> return ProtobufUtil.prependPBMagic(builder.setVersion(version).build().toByteArray());</span> |
| <span class="source-line-no">342</span><span id="line-342"> }</span> |
| <span class="source-line-no">343</span><span id="line-343"></span> |
| <span class="source-line-no">344</span><span id="line-344"> /**</span> |
| <span class="source-line-no">345</span><span id="line-345"> * Verifies current version of file system</span> |
| <span class="source-line-no">346</span><span id="line-346"> * @param fs file system</span> |
| <span class="source-line-no">347</span><span id="line-347"> * @param rootdir root directory of HBase installation</span> |
| <span class="source-line-no">348</span><span id="line-348"> * @param message if true, issues a message on System.out</span> |
| <span class="source-line-no">349</span><span id="line-349"> * @throws IOException if the version file cannot be opened</span> |
| <span class="source-line-no">350</span><span id="line-350"> * @throws DeserializationException if the contents of the version file cannot be parsed</span> |
| <span class="source-line-no">351</span><span id="line-351"> */</span> |
| <span class="source-line-no">352</span><span id="line-352"> public static void checkVersion(FileSystem fs, Path rootdir, boolean message)</span> |
| <span class="source-line-no">353</span><span id="line-353"> throws IOException, DeserializationException {</span> |
| <span class="source-line-no">354</span><span id="line-354"> checkVersion(fs, rootdir, message, 0, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);</span> |
| <span class="source-line-no">355</span><span id="line-355"> }</span> |
| <span class="source-line-no">356</span><span id="line-356"></span> |
| <span class="source-line-no">357</span><span id="line-357"> /**</span> |
| <span class="source-line-no">358</span><span id="line-358"> * Verifies current version of file system</span> |
| <span class="source-line-no">359</span><span id="line-359"> * @param fs file system</span> |
| <span class="source-line-no">360</span><span id="line-360"> * @param rootdir root directory of HBase installation</span> |
| <span class="source-line-no">361</span><span id="line-361"> * @param message if true, issues a message on System.out</span> |
| <span class="source-line-no">362</span><span id="line-362"> * @param wait wait interval</span> |
| <span class="source-line-no">363</span><span id="line-363"> * @param retries number of times to retry</span> |
| <span class="source-line-no">364</span><span id="line-364"> * @throws IOException if the version file cannot be opened</span> |
| <span class="source-line-no">365</span><span id="line-365"> * @throws DeserializationException if the contents of the version file cannot be parsed</span> |
| <span class="source-line-no">366</span><span id="line-366"> */</span> |
| <span class="source-line-no">367</span><span id="line-367"> public static void checkVersion(FileSystem fs, Path rootdir, boolean message, int wait,</span> |
| <span class="source-line-no">368</span><span id="line-368"> int retries) throws IOException, DeserializationException {</span> |
| <span class="source-line-no">369</span><span id="line-369"> String version = getVersion(fs, rootdir);</span> |
| <span class="source-line-no">370</span><span id="line-370"> String msg;</span> |
| <span class="source-line-no">371</span><span id="line-371"> if (version == null) {</span> |
| <span class="source-line-no">372</span><span id="line-372"> if (!metaRegionExists(fs, rootdir)) {</span> |
| <span class="source-line-no">373</span><span id="line-373"> // rootDir is empty (no version file and no root region)</span> |
| <span class="source-line-no">374</span><span id="line-374"> // just create new version file (HBASE-1195)</span> |
| <span class="source-line-no">375</span><span id="line-375"> setVersion(fs, rootdir, wait, retries);</span> |
| <span class="source-line-no">376</span><span id="line-376"> return;</span> |
| <span class="source-line-no">377</span><span id="line-377"> } else {</span> |
| <span class="source-line-no">378</span><span id="line-378"> msg = "hbase.version file is missing. Is your hbase.rootdir valid? "</span> |
| <span class="source-line-no">379</span><span id="line-379"> + "You can restore hbase.version file by running 'HBCK2 filesystem -fix'. "</span> |
| <span class="source-line-no">380</span><span id="line-380"> + "See https://github.com/apache/hbase-operator-tools/tree/master/hbase-hbck2";</span> |
| <span class="source-line-no">381</span><span id="line-381"> }</span> |
| <span class="source-line-no">382</span><span id="line-382"> } else if (version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0) {</span> |
| <span class="source-line-no">383</span><span id="line-383"> return;</span> |
| <span class="source-line-no">384</span><span id="line-384"> } else {</span> |
| <span class="source-line-no">385</span><span id="line-385"> msg = "HBase file layout needs to be upgraded. Current filesystem version is " + version</span> |
| <span class="source-line-no">386</span><span id="line-386"> + " but software requires version " + HConstants.FILE_SYSTEM_VERSION</span> |
| <span class="source-line-no">387</span><span id="line-387"> + ". Consult http://hbase.apache.org/book.html for further information about "</span> |
| <span class="source-line-no">388</span><span id="line-388"> + "upgrading HBase.";</span> |
| <span class="source-line-no">389</span><span id="line-389"> }</span> |
| <span class="source-line-no">390</span><span id="line-390"></span> |
| <span class="source-line-no">391</span><span id="line-391"> // version is deprecated require migration</span> |
| <span class="source-line-no">392</span><span id="line-392"> // Output on stdout so user sees it in terminal.</span> |
| <span class="source-line-no">393</span><span id="line-393"> if (message) {</span> |
| <span class="source-line-no">394</span><span id="line-394"> System.out.println("WARNING! " + msg);</span> |
| <span class="source-line-no">395</span><span id="line-395"> }</span> |
| <span class="source-line-no">396</span><span id="line-396"> throw new FileSystemVersionException(msg);</span> |
| <span class="source-line-no">397</span><span id="line-397"> }</span> |
| <span class="source-line-no">398</span><span id="line-398"></span> |
| <span class="source-line-no">399</span><span id="line-399"> /**</span> |
| <span class="source-line-no">400</span><span id="line-400"> * Sets version of file system</span> |
| <span class="source-line-no">401</span><span id="line-401"> * @param fs filesystem object</span> |
| <span class="source-line-no">402</span><span id="line-402"> * @param rootdir hbase root</span> |
| <span class="source-line-no">403</span><span id="line-403"> * @throws IOException e</span> |
| <span class="source-line-no">404</span><span id="line-404"> */</span> |
| <span class="source-line-no">405</span><span id="line-405"> public static void setVersion(FileSystem fs, Path rootdir) throws IOException {</span> |
| <span class="source-line-no">406</span><span id="line-406"> setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, 0,</span> |
| <span class="source-line-no">407</span><span id="line-407"> HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS);</span> |
| <span class="source-line-no">408</span><span id="line-408"> }</span> |
| <span class="source-line-no">409</span><span id="line-409"></span> |
| <span class="source-line-no">410</span><span id="line-410"> /**</span> |
| <span class="source-line-no">411</span><span id="line-411"> * Sets version of file system</span> |
| <span class="source-line-no">412</span><span id="line-412"> * @param fs filesystem object</span> |
| <span class="source-line-no">413</span><span id="line-413"> * @param rootdir hbase root</span> |
| <span class="source-line-no">414</span><span id="line-414"> * @param wait time to wait for retry</span> |
| <span class="source-line-no">415</span><span id="line-415"> * @param retries number of times to retry before failing</span> |
| <span class="source-line-no">416</span><span id="line-416"> * @throws IOException e</span> |
| <span class="source-line-no">417</span><span id="line-417"> */</span> |
| <span class="source-line-no">418</span><span id="line-418"> public static void setVersion(FileSystem fs, Path rootdir, int wait, int retries)</span> |
| <span class="source-line-no">419</span><span id="line-419"> throws IOException {</span> |
| <span class="source-line-no">420</span><span id="line-420"> setVersion(fs, rootdir, HConstants.FILE_SYSTEM_VERSION, wait, retries);</span> |
| <span class="source-line-no">421</span><span id="line-421"> }</span> |
| <span class="source-line-no">422</span><span id="line-422"></span> |
| <span class="source-line-no">423</span><span id="line-423"> /**</span> |
| <span class="source-line-no">424</span><span id="line-424"> * Sets version of file system</span> |
| <span class="source-line-no">425</span><span id="line-425"> * @param fs filesystem object</span> |
| <span class="source-line-no">426</span><span id="line-426"> * @param rootdir hbase root directory</span> |
| <span class="source-line-no">427</span><span id="line-427"> * @param version version to set</span> |
| <span class="source-line-no">428</span><span id="line-428"> * @param wait time to wait for retry</span> |
| <span class="source-line-no">429</span><span id="line-429"> * @param retries number of times to retry before throwing an IOException</span> |
| <span class="source-line-no">430</span><span id="line-430"> * @throws IOException e</span> |
| <span class="source-line-no">431</span><span id="line-431"> */</span> |
| <span class="source-line-no">432</span><span id="line-432"> public static void setVersion(FileSystem fs, Path rootdir, String version, int wait, int retries)</span> |
| <span class="source-line-no">433</span><span id="line-433"> throws IOException {</span> |
| <span class="source-line-no">434</span><span id="line-434"> Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);</span> |
| <span class="source-line-no">435</span><span id="line-435"> Path tempVersionFile = new Path(rootdir,</span> |
| <span class="source-line-no">436</span><span id="line-436"> HConstants.HBASE_TEMP_DIRECTORY + Path.SEPARATOR + HConstants.VERSION_FILE_NAME);</span> |
| <span class="source-line-no">437</span><span id="line-437"> while (true) {</span> |
| <span class="source-line-no">438</span><span id="line-438"> try {</span> |
| <span class="source-line-no">439</span><span id="line-439"> // Write the version to a temporary file</span> |
| <span class="source-line-no">440</span><span id="line-440"> FSDataOutputStream s = fs.create(tempVersionFile);</span> |
| <span class="source-line-no">441</span><span id="line-441"> try {</span> |
| <span class="source-line-no">442</span><span id="line-442"> s.write(toVersionByteArray(version));</span> |
| <span class="source-line-no">443</span><span id="line-443"> s.close();</span> |
| <span class="source-line-no">444</span><span id="line-444"> s = null;</span> |
| <span class="source-line-no">445</span><span id="line-445"> // Move the temp version file to its normal location. Returns false</span> |
| <span class="source-line-no">446</span><span id="line-446"> // if the rename failed. Throw an IOE in that case.</span> |
| <span class="source-line-no">447</span><span id="line-447"> if (!fs.rename(tempVersionFile, versionFile)) {</span> |
| <span class="source-line-no">448</span><span id="line-448"> throw new IOException("Unable to move temp version file to " + versionFile);</span> |
| <span class="source-line-no">449</span><span id="line-449"> }</span> |
| <span class="source-line-no">450</span><span id="line-450"> } finally {</span> |
| <span class="source-line-no">451</span><span id="line-451"> // Cleaning up the temporary if the rename failed would be trying</span> |
| <span class="source-line-no">452</span><span id="line-452"> // too hard. We'll unconditionally create it again the next time</span> |
| <span class="source-line-no">453</span><span id="line-453"> // through anyway, files are overwritten by default by create().</span> |
| <span class="source-line-no">454</span><span id="line-454"></span> |
| <span class="source-line-no">455</span><span id="line-455"> // Attempt to close the stream on the way out if it is still open.</span> |
| <span class="source-line-no">456</span><span id="line-456"> try {</span> |
| <span class="source-line-no">457</span><span id="line-457"> if (s != null) s.close();</span> |
| <span class="source-line-no">458</span><span id="line-458"> } catch (IOException ignore) {</span> |
| <span class="source-line-no">459</span><span id="line-459"> }</span> |
| <span class="source-line-no">460</span><span id="line-460"> }</span> |
| <span class="source-line-no">461</span><span id="line-461"> LOG.info("Created version file at " + rootdir.toString() + " with version=" + version);</span> |
| <span class="source-line-no">462</span><span id="line-462"> return;</span> |
| <span class="source-line-no">463</span><span id="line-463"> } catch (IOException e) {</span> |
| <span class="source-line-no">464</span><span id="line-464"> if (retries > 0) {</span> |
| <span class="source-line-no">465</span><span id="line-465"> LOG.debug("Unable to create version file at " + rootdir.toString() + ", retrying", e);</span> |
| <span class="source-line-no">466</span><span id="line-466"> fs.delete(versionFile, false);</span> |
| <span class="source-line-no">467</span><span id="line-467"> try {</span> |
| <span class="source-line-no">468</span><span id="line-468"> if (wait > 0) {</span> |
| <span class="source-line-no">469</span><span id="line-469"> Thread.sleep(wait);</span> |
| <span class="source-line-no">470</span><span id="line-470"> }</span> |
| <span class="source-line-no">471</span><span id="line-471"> } catch (InterruptedException ie) {</span> |
| <span class="source-line-no">472</span><span id="line-472"> throw (InterruptedIOException) new InterruptedIOException().initCause(ie);</span> |
| <span class="source-line-no">473</span><span id="line-473"> }</span> |
| <span class="source-line-no">474</span><span id="line-474"> retries--;</span> |
| <span class="source-line-no">475</span><span id="line-475"> } else {</span> |
| <span class="source-line-no">476</span><span id="line-476"> throw e;</span> |
| <span class="source-line-no">477</span><span id="line-477"> }</span> |
| <span class="source-line-no">478</span><span id="line-478"> }</span> |
| <span class="source-line-no">479</span><span id="line-479"> }</span> |
| <span class="source-line-no">480</span><span id="line-480"> }</span> |
| <span class="source-line-no">481</span><span id="line-481"></span> |
| <span class="source-line-no">482</span><span id="line-482"> /**</span> |
| <span class="source-line-no">483</span><span id="line-483"> * Checks that a cluster ID file exists in the HBase root directory</span> |
| <span class="source-line-no">484</span><span id="line-484"> * @param fs the root directory FileSystem</span> |
| <span class="source-line-no">485</span><span id="line-485"> * @param rootdir the HBase root directory in HDFS</span> |
| <span class="source-line-no">486</span><span id="line-486"> * @param wait how long to wait between retries</span> |
| <span class="source-line-no">487</span><span id="line-487"> * @return <code>true</code> if the file exists, otherwise <code>false</code></span> |
| <span class="source-line-no">488</span><span id="line-488"> * @throws IOException if checking the FileSystem fails</span> |
| <span class="source-line-no">489</span><span id="line-489"> */</span> |
| <span class="source-line-no">490</span><span id="line-490"> public static boolean checkClusterIdExists(FileSystem fs, Path rootdir, long wait)</span> |
| <span class="source-line-no">491</span><span id="line-491"> throws IOException {</span> |
| <span class="source-line-no">492</span><span id="line-492"> while (true) {</span> |
| <span class="source-line-no">493</span><span id="line-493"> try {</span> |
| <span class="source-line-no">494</span><span id="line-494"> Path filePath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);</span> |
| <span class="source-line-no">495</span><span id="line-495"> return fs.exists(filePath);</span> |
| <span class="source-line-no">496</span><span id="line-496"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">497</span><span id="line-497"> if (wait > 0L) {</span> |
| <span class="source-line-no">498</span><span id="line-498"> LOG.warn("Unable to check cluster ID file in {}, retrying in {}ms", rootdir, wait, ioe);</span> |
| <span class="source-line-no">499</span><span id="line-499"> try {</span> |
| <span class="source-line-no">500</span><span id="line-500"> Thread.sleep(wait);</span> |
| <span class="source-line-no">501</span><span id="line-501"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">502</span><span id="line-502"> Thread.currentThread().interrupt();</span> |
| <span class="source-line-no">503</span><span id="line-503"> throw (InterruptedIOException) new InterruptedIOException().initCause(e);</span> |
| <span class="source-line-no">504</span><span id="line-504"> }</span> |
| <span class="source-line-no">505</span><span id="line-505"> } else {</span> |
| <span class="source-line-no">506</span><span id="line-506"> throw ioe;</span> |
| <span class="source-line-no">507</span><span id="line-507"> }</span> |
| <span class="source-line-no">508</span><span id="line-508"> }</span> |
| <span class="source-line-no">509</span><span id="line-509"> }</span> |
| <span class="source-line-no">510</span><span id="line-510"> }</span> |
| <span class="source-line-no">511</span><span id="line-511"></span> |
| <span class="source-line-no">512</span><span id="line-512"> /**</span> |
| <span class="source-line-no">513</span><span id="line-513"> * Returns the value of the unique cluster ID stored for this HBase instance.</span> |
| <span class="source-line-no">514</span><span id="line-514"> * @param fs the root directory FileSystem</span> |
| <span class="source-line-no">515</span><span id="line-515"> * @param rootdir the path to the HBase root directory</span> |
| <span class="source-line-no">516</span><span id="line-516"> * @return the unique cluster identifier</span> |
| <span class="source-line-no">517</span><span id="line-517"> * @throws IOException if reading the cluster ID file fails</span> |
| <span class="source-line-no">518</span><span id="line-518"> */</span> |
| <span class="source-line-no">519</span><span id="line-519"> public static ClusterId getClusterId(FileSystem fs, Path rootdir) throws IOException {</span> |
| <span class="source-line-no">520</span><span id="line-520"> Path idPath = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);</span> |
| <span class="source-line-no">521</span><span id="line-521"> ClusterId clusterId = null;</span> |
| <span class="source-line-no">522</span><span id="line-522"> FileStatus status = fs.exists(idPath) ? fs.getFileStatus(idPath) : null;</span> |
| <span class="source-line-no">523</span><span id="line-523"> if (status != null) {</span> |
| <span class="source-line-no">524</span><span id="line-524"> int len = Ints.checkedCast(status.getLen());</span> |
| <span class="source-line-no">525</span><span id="line-525"> byte[] content = new byte[len];</span> |
| <span class="source-line-no">526</span><span id="line-526"> FSDataInputStream in = fs.open(idPath);</span> |
| <span class="source-line-no">527</span><span id="line-527"> try {</span> |
| <span class="source-line-no">528</span><span id="line-528"> in.readFully(content);</span> |
| <span class="source-line-no">529</span><span id="line-529"> } catch (EOFException eof) {</span> |
| <span class="source-line-no">530</span><span id="line-530"> LOG.warn("Cluster ID file {} is empty", idPath);</span> |
| <span class="source-line-no">531</span><span id="line-531"> } finally {</span> |
| <span class="source-line-no">532</span><span id="line-532"> in.close();</span> |
| <span class="source-line-no">533</span><span id="line-533"> }</span> |
| <span class="source-line-no">534</span><span id="line-534"> try {</span> |
| <span class="source-line-no">535</span><span id="line-535"> clusterId = ClusterId.parseFrom(content);</span> |
| <span class="source-line-no">536</span><span id="line-536"> } catch (DeserializationException e) {</span> |
| <span class="source-line-no">537</span><span id="line-537"> throw new IOException("content=" + Bytes.toString(content), e);</span> |
| <span class="source-line-no">538</span><span id="line-538"> }</span> |
| <span class="source-line-no">539</span><span id="line-539"> // If not pb'd, make it so.</span> |
| <span class="source-line-no">540</span><span id="line-540"> if (!ProtobufUtil.isPBMagicPrefix(content)) {</span> |
| <span class="source-line-no">541</span><span id="line-541"> String cid = null;</span> |
| <span class="source-line-no">542</span><span id="line-542"> in = fs.open(idPath);</span> |
| <span class="source-line-no">543</span><span id="line-543"> try {</span> |
| <span class="source-line-no">544</span><span id="line-544"> cid = in.readUTF();</span> |
| <span class="source-line-no">545</span><span id="line-545"> clusterId = new ClusterId(cid);</span> |
| <span class="source-line-no">546</span><span id="line-546"> } catch (EOFException eof) {</span> |
| <span class="source-line-no">547</span><span id="line-547"> LOG.warn("Cluster ID file {} is empty", idPath);</span> |
| <span class="source-line-no">548</span><span id="line-548"> } finally {</span> |
| <span class="source-line-no">549</span><span id="line-549"> in.close();</span> |
| <span class="source-line-no">550</span><span id="line-550"> }</span> |
| <span class="source-line-no">551</span><span id="line-551"> rewriteAsPb(fs, rootdir, idPath, clusterId);</span> |
| <span class="source-line-no">552</span><span id="line-552"> }</span> |
| <span class="source-line-no">553</span><span id="line-553"> return clusterId;</span> |
| <span class="source-line-no">554</span><span id="line-554"> } else {</span> |
| <span class="source-line-no">555</span><span id="line-555"> LOG.warn("Cluster ID file does not exist at {}", idPath);</span> |
| <span class="source-line-no">556</span><span id="line-556"> }</span> |
| <span class="source-line-no">557</span><span id="line-557"> return clusterId;</span> |
| <span class="source-line-no">558</span><span id="line-558"> }</span> |
| <span class="source-line-no">559</span><span id="line-559"></span> |
| <span class="source-line-no">560</span><span id="line-560"> /**</span> |
| <span class="source-line-no">561</span><span id="line-561"> * */</span> |
| <span class="source-line-no">562</span><span id="line-562"> private static void rewriteAsPb(final FileSystem fs, final Path rootdir, final Path p,</span> |
| <span class="source-line-no">563</span><span id="line-563"> final ClusterId cid) throws IOException {</span> |
| <span class="source-line-no">564</span><span id="line-564"> // Rewrite the file as pb. Move aside the old one first, write new</span> |
| <span class="source-line-no">565</span><span id="line-565"> // then delete the moved-aside file.</span> |
| <span class="source-line-no">566</span><span id="line-566"> Path movedAsideName = new Path(p + "." + EnvironmentEdgeManager.currentTime());</span> |
| <span class="source-line-no">567</span><span id="line-567"> if (!fs.rename(p, movedAsideName)) throw new IOException("Failed rename of " + p);</span> |
| <span class="source-line-no">568</span><span id="line-568"> setClusterId(fs, rootdir, cid, 100);</span> |
| <span class="source-line-no">569</span><span id="line-569"> if (!fs.delete(movedAsideName, false)) {</span> |
| <span class="source-line-no">570</span><span id="line-570"> throw new IOException("Failed delete of " + movedAsideName);</span> |
| <span class="source-line-no">571</span><span id="line-571"> }</span> |
| <span class="source-line-no">572</span><span id="line-572"> LOG.debug("Rewrote the hbase.id file as pb");</span> |
| <span class="source-line-no">573</span><span id="line-573"> }</span> |
| <span class="source-line-no">574</span><span id="line-574"></span> |
| <span class="source-line-no">575</span><span id="line-575"> /**</span> |
| <span class="source-line-no">576</span><span id="line-576"> * Writes a new unique identifier for this cluster to the "hbase.id" file in the HBase root</span> |
| <span class="source-line-no">577</span><span id="line-577"> * directory. If any operations on the ID file fails, and {@code wait} is a positive value, the</span> |
| <span class="source-line-no">578</span><span id="line-578"> * method will retry to produce the ID file until the thread is forcibly interrupted.</span> |
| <span class="source-line-no">579</span><span id="line-579"> * @param fs the root directory FileSystem</span> |
| <span class="source-line-no">580</span><span id="line-580"> * @param rootdir the path to the HBase root directory</span> |
| <span class="source-line-no">581</span><span id="line-581"> * @param clusterId the unique identifier to store</span> |
| <span class="source-line-no">582</span><span id="line-582"> * @param wait how long (in milliseconds) to wait between retries</span> |
| <span class="source-line-no">583</span><span id="line-583"> * @throws IOException if writing to the FileSystem fails and no wait value</span> |
| <span class="source-line-no">584</span><span id="line-584"> */</span> |
| <span class="source-line-no">585</span><span id="line-585"> public static void setClusterId(final FileSystem fs, final Path rootdir,</span> |
| <span class="source-line-no">586</span><span id="line-586"> final ClusterId clusterId, final long wait) throws IOException {</span> |
| <span class="source-line-no">587</span><span id="line-587"></span> |
| <span class="source-line-no">588</span><span id="line-588"> final Path idFile = new Path(rootdir, HConstants.CLUSTER_ID_FILE_NAME);</span> |
| <span class="source-line-no">589</span><span id="line-589"> final Path tempDir = new Path(rootdir, HConstants.HBASE_TEMP_DIRECTORY);</span> |
| <span class="source-line-no">590</span><span id="line-590"> final Path tempIdFile = new Path(tempDir, HConstants.CLUSTER_ID_FILE_NAME);</span> |
| <span class="source-line-no">591</span><span id="line-591"></span> |
| <span class="source-line-no">592</span><span id="line-592"> LOG.debug("Create cluster ID file [{}] with ID: {}", idFile, clusterId);</span> |
| <span class="source-line-no">593</span><span id="line-593"></span> |
| <span class="source-line-no">594</span><span id="line-594"> while (true) {</span> |
| <span class="source-line-no">595</span><span id="line-595"> Optional<IOException> failure = Optional.empty();</span> |
| <span class="source-line-no">596</span><span id="line-596"></span> |
| <span class="source-line-no">597</span><span id="line-597"> LOG.debug("Write the cluster ID file to a temporary location: {}", tempIdFile);</span> |
| <span class="source-line-no">598</span><span id="line-598"> try (FSDataOutputStream s = fs.create(tempIdFile)) {</span> |
| <span class="source-line-no">599</span><span id="line-599"> s.write(clusterId.toByteArray());</span> |
| <span class="source-line-no">600</span><span id="line-600"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">601</span><span id="line-601"> failure = Optional.of(ioe);</span> |
| <span class="source-line-no">602</span><span id="line-602"> }</span> |
| <span class="source-line-no">603</span><span id="line-603"></span> |
| <span class="source-line-no">604</span><span id="line-604"> if (!failure.isPresent()) {</span> |
| <span class="source-line-no">605</span><span id="line-605"> try {</span> |
| <span class="source-line-no">606</span><span id="line-606"> LOG.debug("Move the temporary cluster ID file to its target location [{}]:[{}]",</span> |
| <span class="source-line-no">607</span><span id="line-607"> tempIdFile, idFile);</span> |
| <span class="source-line-no">608</span><span id="line-608"></span> |
| <span class="source-line-no">609</span><span id="line-609"> if (!fs.rename(tempIdFile, idFile)) {</span> |
| <span class="source-line-no">610</span><span id="line-610"> failure =</span> |
| <span class="source-line-no">611</span><span id="line-611"> Optional.of(new IOException("Unable to move temp cluster ID file to " + idFile));</span> |
| <span class="source-line-no">612</span><span id="line-612"> }</span> |
| <span class="source-line-no">613</span><span id="line-613"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">614</span><span id="line-614"> failure = Optional.of(ioe);</span> |
| <span class="source-line-no">615</span><span id="line-615"> }</span> |
| <span class="source-line-no">616</span><span id="line-616"> }</span> |
| <span class="source-line-no">617</span><span id="line-617"></span> |
| <span class="source-line-no">618</span><span id="line-618"> if (failure.isPresent()) {</span> |
| <span class="source-line-no">619</span><span id="line-619"> final IOException cause = failure.get();</span> |
| <span class="source-line-no">620</span><span id="line-620"> if (wait > 0L) {</span> |
| <span class="source-line-no">621</span><span id="line-621"> LOG.warn("Unable to create cluster ID file in {}, retrying in {}ms", rootdir, wait,</span> |
| <span class="source-line-no">622</span><span id="line-622"> cause);</span> |
| <span class="source-line-no">623</span><span id="line-623"> try {</span> |
| <span class="source-line-no">624</span><span id="line-624"> Thread.sleep(wait);</span> |
| <span class="source-line-no">625</span><span id="line-625"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">626</span><span id="line-626"> Thread.currentThread().interrupt();</span> |
| <span class="source-line-no">627</span><span id="line-627"> throw (InterruptedIOException) new InterruptedIOException().initCause(e);</span> |
| <span class="source-line-no">628</span><span id="line-628"> }</span> |
| <span class="source-line-no">629</span><span id="line-629"> continue;</span> |
| <span class="source-line-no">630</span><span id="line-630"> } else {</span> |
| <span class="source-line-no">631</span><span id="line-631"> throw cause;</span> |
| <span class="source-line-no">632</span><span id="line-632"> }</span> |
| <span class="source-line-no">633</span><span id="line-633"> } else {</span> |
| <span class="source-line-no">634</span><span id="line-634"> return;</span> |
| <span class="source-line-no">635</span><span id="line-635"> }</span> |
| <span class="source-line-no">636</span><span id="line-636"> }</span> |
| <span class="source-line-no">637</span><span id="line-637"> }</span> |
| <span class="source-line-no">638</span><span id="line-638"></span> |
| <span class="source-line-no">639</span><span id="line-639"> /**</span> |
| <span class="source-line-no">640</span><span id="line-640"> * If DFS, check safe mode and if so, wait until we clear it.</span> |
| <span class="source-line-no">641</span><span id="line-641"> * @param conf configuration</span> |
| <span class="source-line-no">642</span><span id="line-642"> * @param wait Sleep between retries</span> |
| <span class="source-line-no">643</span><span id="line-643"> * @throws IOException e</span> |
| <span class="source-line-no">644</span><span id="line-644"> */</span> |
| <span class="source-line-no">645</span><span id="line-645"> public static void waitOnSafeMode(final Configuration conf, final long wait) throws IOException {</span> |
| <span class="source-line-no">646</span><span id="line-646"> FileSystem fs = FileSystem.get(conf);</span> |
| <span class="source-line-no">647</span><span id="line-647"> if (!(fs instanceof DistributedFileSystem)) return;</span> |
| <span class="source-line-no">648</span><span id="line-648"> DistributedFileSystem dfs = (DistributedFileSystem) fs;</span> |
| <span class="source-line-no">649</span><span id="line-649"> // Make sure dfs is not in safe mode</span> |
| <span class="source-line-no">650</span><span id="line-650"> while (isInSafeMode(dfs)) {</span> |
| <span class="source-line-no">651</span><span id="line-651"> LOG.info("Waiting for dfs to exit safe mode...");</span> |
| <span class="source-line-no">652</span><span id="line-652"> try {</span> |
| <span class="source-line-no">653</span><span id="line-653"> Thread.sleep(wait);</span> |
| <span class="source-line-no">654</span><span id="line-654"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">655</span><span id="line-655"> Thread.currentThread().interrupt();</span> |
| <span class="source-line-no">656</span><span id="line-656"> throw (InterruptedIOException) new InterruptedIOException().initCause(e);</span> |
| <span class="source-line-no">657</span><span id="line-657"> }</span> |
| <span class="source-line-no">658</span><span id="line-658"> }</span> |
| <span class="source-line-no">659</span><span id="line-659"> }</span> |
| <span class="source-line-no">660</span><span id="line-660"></span> |
| <span class="source-line-no">661</span><span id="line-661"> /**</span> |
| <span class="source-line-no">662</span><span id="line-662"> * Checks if meta region exists</span> |
| <span class="source-line-no">663</span><span id="line-663"> * @param fs file system</span> |
| <span class="source-line-no">664</span><span id="line-664"> * @param rootDir root directory of HBase installation</span> |
| <span class="source-line-no">665</span><span id="line-665"> * @return true if exists</span> |
| <span class="source-line-no">666</span><span id="line-666"> */</span> |
| <span class="source-line-no">667</span><span id="line-667"> public static boolean metaRegionExists(FileSystem fs, Path rootDir) throws IOException {</span> |
| <span class="source-line-no">668</span><span id="line-668"> Path metaRegionDir = getRegionDirFromRootDir(rootDir, RegionInfoBuilder.FIRST_META_REGIONINFO);</span> |
| <span class="source-line-no">669</span><span id="line-669"> return fs.exists(metaRegionDir);</span> |
| <span class="source-line-no">670</span><span id="line-670"> }</span> |
| <span class="source-line-no">671</span><span id="line-671"></span> |
| <span class="source-line-no">672</span><span id="line-672"> /**</span> |
| <span class="source-line-no">673</span><span id="line-673"> * Compute HDFS block distribution of a given HdfsDataInputStream. All HdfsDataInputStreams are</span> |
| <span class="source-line-no">674</span><span id="line-674"> * backed by a series of LocatedBlocks, which are fetched periodically from the namenode. This</span> |
| <span class="source-line-no">675</span><span id="line-675"> * method retrieves those blocks from the input stream and uses them to calculate</span> |
| <span class="source-line-no">676</span><span id="line-676"> * HDFSBlockDistribution. The underlying method in DFSInputStream does attempt to use locally</span> |
| <span class="source-line-no">677</span><span id="line-677"> * cached blocks, but may hit the namenode if the cache is determined to be incomplete. The method</span> |
| <span class="source-line-no">678</span><span id="line-678"> * also involves making copies of all LocatedBlocks rather than return the underlying blocks</span> |
| <span class="source-line-no">679</span><span id="line-679"> * themselves.</span> |
| <span class="source-line-no">680</span><span id="line-680"> */</span> |
| <span class="source-line-no">681</span><span id="line-681"> static public HDFSBlocksDistribution</span> |
| <span class="source-line-no">682</span><span id="line-682"> computeHDFSBlocksDistribution(HdfsDataInputStream inputStream) throws IOException {</span> |
| <span class="source-line-no">683</span><span id="line-683"> List<LocatedBlock> blocks = inputStream.getAllBlocks();</span> |
| <span class="source-line-no">684</span><span id="line-684"> HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();</span> |
| <span class="source-line-no">685</span><span id="line-685"> for (LocatedBlock block : blocks) {</span> |
| <span class="source-line-no">686</span><span id="line-686"> String[] hosts = getHostsForLocations(block);</span> |
| <span class="source-line-no">687</span><span id="line-687"> long len = block.getBlockSize();</span> |
| <span class="source-line-no">688</span><span id="line-688"> StorageType[] storageTypes = block.getStorageTypes();</span> |
| <span class="source-line-no">689</span><span id="line-689"> blocksDistribution.addHostsAndBlockWeight(hosts, len, storageTypes);</span> |
| <span class="source-line-no">690</span><span id="line-690"> }</span> |
| <span class="source-line-no">691</span><span id="line-691"> return blocksDistribution;</span> |
| <span class="source-line-no">692</span><span id="line-692"> }</span> |
| <span class="source-line-no">693</span><span id="line-693"></span> |
| <span class="source-line-no">694</span><span id="line-694"> private static String[] getHostsForLocations(LocatedBlock block) {</span> |
| <span class="source-line-no">695</span><span id="line-695"> DatanodeInfo[] locations = getLocatedBlockLocations(block);</span> |
| <span class="source-line-no">696</span><span id="line-696"> String[] hosts = new String[locations.length];</span> |
| <span class="source-line-no">697</span><span id="line-697"> for (int i = 0; i < hosts.length; i++) {</span> |
| <span class="source-line-no">698</span><span id="line-698"> hosts[i] = locations[i].getHostName();</span> |
| <span class="source-line-no">699</span><span id="line-699"> }</span> |
| <span class="source-line-no">700</span><span id="line-700"> return hosts;</span> |
| <span class="source-line-no">701</span><span id="line-701"> }</span> |
| <span class="source-line-no">702</span><span id="line-702"></span> |
| <span class="source-line-no">703</span><span id="line-703"> /**</span> |
| <span class="source-line-no">704</span><span id="line-704"> * Compute HDFS blocks distribution of a given file, or a portion of the file</span> |
| <span class="source-line-no">705</span><span id="line-705"> * @param fs file system</span> |
| <span class="source-line-no">706</span><span id="line-706"> * @param status file status of the file</span> |
| <span class="source-line-no">707</span><span id="line-707"> * @param start start position of the portion</span> |
| <span class="source-line-no">708</span><span id="line-708"> * @param length length of the portion</span> |
| <span class="source-line-no">709</span><span id="line-709"> * @return The HDFS blocks distribution</span> |
| <span class="source-line-no">710</span><span id="line-710"> */</span> |
| <span class="source-line-no">711</span><span id="line-711"> static public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs,</span> |
| <span class="source-line-no">712</span><span id="line-712"> FileStatus status, long start, long length) throws IOException {</span> |
| <span class="source-line-no">713</span><span id="line-713"> HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution();</span> |
| <span class="source-line-no">714</span><span id="line-714"> BlockLocation[] blockLocations = fs.getFileBlockLocations(status, start, length);</span> |
| <span class="source-line-no">715</span><span id="line-715"> addToHDFSBlocksDistribution(blocksDistribution, blockLocations);</span> |
| <span class="source-line-no">716</span><span id="line-716"> return blocksDistribution;</span> |
| <span class="source-line-no">717</span><span id="line-717"> }</span> |
| <span class="source-line-no">718</span><span id="line-718"></span> |
| <span class="source-line-no">719</span><span id="line-719"> /**</span> |
| <span class="source-line-no">720</span><span id="line-720"> * Update blocksDistribution with blockLocations</span> |
| <span class="source-line-no">721</span><span id="line-721"> * @param blocksDistribution the hdfs blocks distribution</span> |
| <span class="source-line-no">722</span><span id="line-722"> * @param blockLocations an array containing block location</span> |
| <span class="source-line-no">723</span><span id="line-723"> */</span> |
| <span class="source-line-no">724</span><span id="line-724"> static public void addToHDFSBlocksDistribution(HDFSBlocksDistribution blocksDistribution,</span> |
| <span class="source-line-no">725</span><span id="line-725"> BlockLocation[] blockLocations) throws IOException {</span> |
| <span class="source-line-no">726</span><span id="line-726"> for (BlockLocation bl : blockLocations) {</span> |
| <span class="source-line-no">727</span><span id="line-727"> String[] hosts = bl.getHosts();</span> |
| <span class="source-line-no">728</span><span id="line-728"> long len = bl.getLength();</span> |
| <span class="source-line-no">729</span><span id="line-729"> StorageType[] storageTypes = bl.getStorageTypes();</span> |
| <span class="source-line-no">730</span><span id="line-730"> blocksDistribution.addHostsAndBlockWeight(hosts, len, storageTypes);</span> |
| <span class="source-line-no">731</span><span id="line-731"> }</span> |
| <span class="source-line-no">732</span><span id="line-732"> }</span> |
| <span class="source-line-no">733</span><span id="line-733"></span> |
| <span class="source-line-no">734</span><span id="line-734"> // TODO move this method OUT of FSUtils. No dependencies to HMaster</span> |
| <span class="source-line-no">735</span><span id="line-735"> /**</span> |
| <span class="source-line-no">736</span><span id="line-736"> * Returns the total overall fragmentation percentage. Includes hbase:meta and -ROOT- as well.</span> |
| <span class="source-line-no">737</span><span id="line-737"> * @param master The master defining the HBase root and file system</span> |
| <span class="source-line-no">738</span><span id="line-738"> * @return A map for each table and its percentage (never null)</span> |
| <span class="source-line-no">739</span><span id="line-739"> * @throws IOException When scanning the directory fails</span> |
| <span class="source-line-no">740</span><span id="line-740"> */</span> |
| <span class="source-line-no">741</span><span id="line-741"> public static int getTotalTableFragmentation(final HMaster master) throws IOException {</span> |
| <span class="source-line-no">742</span><span id="line-742"> Map<String, Integer> map = getTableFragmentation(master);</span> |
| <span class="source-line-no">743</span><span id="line-743"> return map.isEmpty() ? -1 : map.get("-TOTAL-");</span> |
| <span class="source-line-no">744</span><span id="line-744"> }</span> |
| <span class="source-line-no">745</span><span id="line-745"></span> |
| <span class="source-line-no">746</span><span id="line-746"> /**</span> |
| <span class="source-line-no">747</span><span id="line-747"> * Runs through the HBase rootdir and checks how many stores for each table have more than one</span> |
| <span class="source-line-no">748</span><span id="line-748"> * file in them. Checks -ROOT- and hbase:meta too. The total percentage across all tables is</span> |
| <span class="source-line-no">749</span><span id="line-749"> * stored under the special key "-TOTAL-".</span> |
| <span class="source-line-no">750</span><span id="line-750"> * @param master The master defining the HBase root and file system.</span> |
| <span class="source-line-no">751</span><span id="line-751"> * @return A map for each table and its percentage (never null).</span> |
| <span class="source-line-no">752</span><span id="line-752"> * @throws IOException When scanning the directory fails.</span> |
| <span class="source-line-no">753</span><span id="line-753"> */</span> |
| <span class="source-line-no">754</span><span id="line-754"> public static Map<String, Integer> getTableFragmentation(final HMaster master)</span> |
| <span class="source-line-no">755</span><span id="line-755"> throws IOException {</span> |
| <span class="source-line-no">756</span><span id="line-756"> Path path = CommonFSUtils.getRootDir(master.getConfiguration());</span> |
| <span class="source-line-no">757</span><span id="line-757"> // since HMaster.getFileSystem() is package private</span> |
| <span class="source-line-no">758</span><span id="line-758"> FileSystem fs = path.getFileSystem(master.getConfiguration());</span> |
| <span class="source-line-no">759</span><span id="line-759"> return getTableFragmentation(fs, path);</span> |
| <span class="source-line-no">760</span><span id="line-760"> }</span> |
| <span class="source-line-no">761</span><span id="line-761"></span> |
| <span class="source-line-no">762</span><span id="line-762"> /**</span> |
| <span class="source-line-no">763</span><span id="line-763"> * Runs through the HBase rootdir and checks how many stores for each table have more than one</span> |
| <span class="source-line-no">764</span><span id="line-764"> * file in them. Checks -ROOT- and hbase:meta too. The total percentage across all tables is</span> |
| <span class="source-line-no">765</span><span id="line-765"> * stored under the special key "-TOTAL-".</span> |
| <span class="source-line-no">766</span><span id="line-766"> * @param fs The file system to use</span> |
| <span class="source-line-no">767</span><span id="line-767"> * @param hbaseRootDir The root directory to scan</span> |
| <span class="source-line-no">768</span><span id="line-768"> * @return A map for each table and its percentage (never null)</span> |
| <span class="source-line-no">769</span><span id="line-769"> * @throws IOException When scanning the directory fails</span> |
| <span class="source-line-no">770</span><span id="line-770"> */</span> |
| <span class="source-line-no">771</span><span id="line-771"> public static Map<String, Integer> getTableFragmentation(final FileSystem fs,</span> |
| <span class="source-line-no">772</span><span id="line-772"> final Path hbaseRootDir) throws IOException {</span> |
| <span class="source-line-no">773</span><span id="line-773"> Map<String, Integer> frags = new HashMap<>();</span> |
| <span class="source-line-no">774</span><span id="line-774"> int cfCountTotal = 0;</span> |
| <span class="source-line-no">775</span><span id="line-775"> int cfFragTotal = 0;</span> |
| <span class="source-line-no">776</span><span id="line-776"> PathFilter regionFilter = new RegionDirFilter(fs);</span> |
| <span class="source-line-no">777</span><span id="line-777"> PathFilter familyFilter = new FamilyDirFilter(fs);</span> |
| <span class="source-line-no">778</span><span id="line-778"> List<Path> tableDirs = getTableDirs(fs, hbaseRootDir);</span> |
| <span class="source-line-no">779</span><span id="line-779"> for (Path d : tableDirs) {</span> |
| <span class="source-line-no">780</span><span id="line-780"> int cfCount = 0;</span> |
| <span class="source-line-no">781</span><span id="line-781"> int cfFrag = 0;</span> |
| <span class="source-line-no">782</span><span id="line-782"> FileStatus[] regionDirs = fs.listStatus(d, regionFilter);</span> |
| <span class="source-line-no">783</span><span id="line-783"> for (FileStatus regionDir : regionDirs) {</span> |
| <span class="source-line-no">784</span><span id="line-784"> Path dd = regionDir.getPath();</span> |
| <span class="source-line-no">785</span><span id="line-785"> // else its a region name, now look in region for families</span> |
| <span class="source-line-no">786</span><span id="line-786"> FileStatus[] familyDirs = fs.listStatus(dd, familyFilter);</span> |
| <span class="source-line-no">787</span><span id="line-787"> for (FileStatus familyDir : familyDirs) {</span> |
| <span class="source-line-no">788</span><span id="line-788"> cfCount++;</span> |
| <span class="source-line-no">789</span><span id="line-789"> cfCountTotal++;</span> |
| <span class="source-line-no">790</span><span id="line-790"> Path family = familyDir.getPath();</span> |
| <span class="source-line-no">791</span><span id="line-791"> // now in family make sure only one file</span> |
| <span class="source-line-no">792</span><span id="line-792"> FileStatus[] familyStatus = fs.listStatus(family);</span> |
| <span class="source-line-no">793</span><span id="line-793"> if (familyStatus.length > 1) {</span> |
| <span class="source-line-no">794</span><span id="line-794"> cfFrag++;</span> |
| <span class="source-line-no">795</span><span id="line-795"> cfFragTotal++;</span> |
| <span class="source-line-no">796</span><span id="line-796"> }</span> |
| <span class="source-line-no">797</span><span id="line-797"> }</span> |
| <span class="source-line-no">798</span><span id="line-798"> }</span> |
| <span class="source-line-no">799</span><span id="line-799"> // compute percentage per table and store in result list</span> |
| <span class="source-line-no">800</span><span id="line-800"> frags.put(CommonFSUtils.getTableName(d).getNameAsString(),</span> |
| <span class="source-line-no">801</span><span id="line-801"> cfCount == 0 ? 0 : Math.round((float) cfFrag / cfCount * 100));</span> |
| <span class="source-line-no">802</span><span id="line-802"> }</span> |
| <span class="source-line-no">803</span><span id="line-803"> // set overall percentage for all tables</span> |
| <span class="source-line-no">804</span><span id="line-804"> frags.put("-TOTAL-",</span> |
| <span class="source-line-no">805</span><span id="line-805"> cfCountTotal == 0 ? 0 : Math.round((float) cfFragTotal / cfCountTotal * 100));</span> |
| <span class="source-line-no">806</span><span id="line-806"> return frags;</span> |
| <span class="source-line-no">807</span><span id="line-807"> }</span> |
| <span class="source-line-no">808</span><span id="line-808"></span> |
| <span class="source-line-no">809</span><span id="line-809"> public static void renameFile(FileSystem fs, Path src, Path dst) throws IOException {</span> |
| <span class="source-line-no">810</span><span id="line-810"> if (fs.exists(dst) && !fs.delete(dst, false)) {</span> |
| <span class="source-line-no">811</span><span id="line-811"> throw new IOException("Can not delete " + dst);</span> |
| <span class="source-line-no">812</span><span id="line-812"> }</span> |
| <span class="source-line-no">813</span><span id="line-813"> if (!fs.rename(src, dst)) {</span> |
| <span class="source-line-no">814</span><span id="line-814"> throw new IOException("Can not rename from " + src + " to " + dst);</span> |
| <span class="source-line-no">815</span><span id="line-815"> }</span> |
| <span class="source-line-no">816</span><span id="line-816"> }</span> |
| <span class="source-line-no">817</span><span id="line-817"></span> |
| <span class="source-line-no">818</span><span id="line-818"> /**</span> |
| <span class="source-line-no">819</span><span id="line-819"> * A {@link PathFilter} that returns only regular files.</span> |
| <span class="source-line-no">820</span><span id="line-820"> */</span> |
| <span class="source-line-no">821</span><span id="line-821"> static class FileFilter extends AbstractFileStatusFilter {</span> |
| <span class="source-line-no">822</span><span id="line-822"> private final FileSystem fs;</span> |
| <span class="source-line-no">823</span><span id="line-823"></span> |
| <span class="source-line-no">824</span><span id="line-824"> public FileFilter(final FileSystem fs) {</span> |
| <span class="source-line-no">825</span><span id="line-825"> this.fs = fs;</span> |
| <span class="source-line-no">826</span><span id="line-826"> }</span> |
| <span class="source-line-no">827</span><span id="line-827"></span> |
| <span class="source-line-no">828</span><span id="line-828"> @Override</span> |
| <span class="source-line-no">829</span><span id="line-829"> protected boolean accept(Path p, @CheckForNull Boolean isDir) {</span> |
| <span class="source-line-no">830</span><span id="line-830"> try {</span> |
| <span class="source-line-no">831</span><span id="line-831"> return isFile(fs, isDir, p);</span> |
| <span class="source-line-no">832</span><span id="line-832"> } catch (IOException e) {</span> |
| <span class="source-line-no">833</span><span id="line-833"> LOG.warn("Unable to verify if path={} is a regular file", p, e);</span> |
| <span class="source-line-no">834</span><span id="line-834"> return false;</span> |
| <span class="source-line-no">835</span><span id="line-835"> }</span> |
| <span class="source-line-no">836</span><span id="line-836"> }</span> |
| <span class="source-line-no">837</span><span id="line-837"> }</span> |
| <span class="source-line-no">838</span><span id="line-838"></span> |
| <span class="source-line-no">839</span><span id="line-839"> /**</span> |
| <span class="source-line-no">840</span><span id="line-840"> * Directory filter that doesn't include any of the directories in the specified blacklist</span> |
| <span class="source-line-no">841</span><span id="line-841"> */</span> |
| <span class="source-line-no">842</span><span id="line-842"> public static class BlackListDirFilter extends AbstractFileStatusFilter {</span> |
| <span class="source-line-no">843</span><span id="line-843"> private final FileSystem fs;</span> |
| <span class="source-line-no">844</span><span id="line-844"> private List<String> blacklist;</span> |
| <span class="source-line-no">845</span><span id="line-845"></span> |
| <span class="source-line-no">846</span><span id="line-846"> /**</span> |
| <span class="source-line-no">847</span><span id="line-847"> * Create a filter on the givem filesystem with the specified blacklist</span> |
| <span class="source-line-no">848</span><span id="line-848"> * @param fs filesystem to filter</span> |
| <span class="source-line-no">849</span><span id="line-849"> * @param directoryNameBlackList list of the names of the directories to filter. If</span> |
| <span class="source-line-no">850</span><span id="line-850"> * <tt>null</tt>, all directories are returned</span> |
| <span class="source-line-no">851</span><span id="line-851"> */</span> |
| <span class="source-line-no">852</span><span id="line-852"> @SuppressWarnings("unchecked")</span> |
| <span class="source-line-no">853</span><span id="line-853"> public BlackListDirFilter(final FileSystem fs, final List<String> directoryNameBlackList) {</span> |
| <span class="source-line-no">854</span><span id="line-854"> this.fs = fs;</span> |
| <span class="source-line-no">855</span><span id="line-855"> blacklist = (List<String>) (directoryNameBlackList == null</span> |
| <span class="source-line-no">856</span><span id="line-856"> ? Collections.emptyList()</span> |
| <span class="source-line-no">857</span><span id="line-857"> : directoryNameBlackList);</span> |
| <span class="source-line-no">858</span><span id="line-858"> }</span> |
| <span class="source-line-no">859</span><span id="line-859"></span> |
| <span class="source-line-no">860</span><span id="line-860"> @Override</span> |
| <span class="source-line-no">861</span><span id="line-861"> protected boolean accept(Path p, @CheckForNull Boolean isDir) {</span> |
| <span class="source-line-no">862</span><span id="line-862"> if (!isValidName(p.getName())) {</span> |
| <span class="source-line-no">863</span><span id="line-863"> return false;</span> |
| <span class="source-line-no">864</span><span id="line-864"> }</span> |
| <span class="source-line-no">865</span><span id="line-865"></span> |
| <span class="source-line-no">866</span><span id="line-866"> try {</span> |
| <span class="source-line-no">867</span><span id="line-867"> return isDirectory(fs, isDir, p);</span> |
| <span class="source-line-no">868</span><span id="line-868"> } catch (IOException e) {</span> |
| <span class="source-line-no">869</span><span id="line-869"> LOG.warn("An error occurred while verifying if [{}] is a valid directory."</span> |
| <span class="source-line-no">870</span><span id="line-870"> + " Returning 'not valid' and continuing.", p, e);</span> |
| <span class="source-line-no">871</span><span id="line-871"> return false;</span> |
| <span class="source-line-no">872</span><span id="line-872"> }</span> |
| <span class="source-line-no">873</span><span id="line-873"> }</span> |
| <span class="source-line-no">874</span><span id="line-874"></span> |
| <span class="source-line-no">875</span><span id="line-875"> protected boolean isValidName(final String name) {</span> |
| <span class="source-line-no">876</span><span id="line-876"> return !blacklist.contains(name);</span> |
| <span class="source-line-no">877</span><span id="line-877"> }</span> |
| <span class="source-line-no">878</span><span id="line-878"> }</span> |
| <span class="source-line-no">879</span><span id="line-879"></span> |
| <span class="source-line-no">880</span><span id="line-880"> /**</span> |
| <span class="source-line-no">881</span><span id="line-881"> * A {@link PathFilter} that only allows directories.</span> |
| <span class="source-line-no">882</span><span id="line-882"> */</span> |
| <span class="source-line-no">883</span><span id="line-883"> public static class DirFilter extends BlackListDirFilter {</span> |
| <span class="source-line-no">884</span><span id="line-884"></span> |
| <span class="source-line-no">885</span><span id="line-885"> public DirFilter(FileSystem fs) {</span> |
| <span class="source-line-no">886</span><span id="line-886"> super(fs, null);</span> |
| <span class="source-line-no">887</span><span id="line-887"> }</span> |
| <span class="source-line-no">888</span><span id="line-888"> }</span> |
| <span class="source-line-no">889</span><span id="line-889"></span> |
| <span class="source-line-no">890</span><span id="line-890"> /**</span> |
| <span class="source-line-no">891</span><span id="line-891"> * A {@link PathFilter} that returns usertable directories. To get all directories use the</span> |
| <span class="source-line-no">892</span><span id="line-892"> * {@link BlackListDirFilter} with a <tt>null</tt> blacklist</span> |
| <span class="source-line-no">893</span><span id="line-893"> */</span> |
| <span class="source-line-no">894</span><span id="line-894"> public static class UserTableDirFilter extends BlackListDirFilter {</span> |
| <span class="source-line-no">895</span><span id="line-895"> public UserTableDirFilter(FileSystem fs) {</span> |
| <span class="source-line-no">896</span><span id="line-896"> super(fs, HConstants.HBASE_NON_TABLE_DIRS);</span> |
| <span class="source-line-no">897</span><span id="line-897"> }</span> |
| <span class="source-line-no">898</span><span id="line-898"></span> |
| <span class="source-line-no">899</span><span id="line-899"> @Override</span> |
| <span class="source-line-no">900</span><span id="line-900"> protected boolean isValidName(final String name) {</span> |
| <span class="source-line-no">901</span><span id="line-901"> if (!super.isValidName(name)) return false;</span> |
| <span class="source-line-no">902</span><span id="line-902"></span> |
| <span class="source-line-no">903</span><span id="line-903"> try {</span> |
| <span class="source-line-no">904</span><span id="line-904"> TableName.isLegalTableQualifierName(Bytes.toBytes(name));</span> |
| <span class="source-line-no">905</span><span id="line-905"> } catch (IllegalArgumentException e) {</span> |
| <span class="source-line-no">906</span><span id="line-906"> LOG.info("Invalid table name: {}", name);</span> |
| <span class="source-line-no">907</span><span id="line-907"> return false;</span> |
| <span class="source-line-no">908</span><span id="line-908"> }</span> |
| <span class="source-line-no">909</span><span id="line-909"> return true;</span> |
| <span class="source-line-no">910</span><span id="line-910"> }</span> |
| <span class="source-line-no">911</span><span id="line-911"> }</span> |
| <span class="source-line-no">912</span><span id="line-912"></span> |
| <span class="source-line-no">913</span><span id="line-913"> public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)</span> |
| <span class="source-line-no">914</span><span id="line-914"> throws IOException {</span> |
| <span class="source-line-no">915</span><span id="line-915"> List<Path> tableDirs = new ArrayList<>();</span> |
| <span class="source-line-no">916</span><span id="line-916"> Path baseNamespaceDir = new Path(rootdir, HConstants.BASE_NAMESPACE_DIR);</span> |
| <span class="source-line-no">917</span><span id="line-917"> if (fs.exists(baseNamespaceDir)) {</span> |
| <span class="source-line-no">918</span><span id="line-918"> for (FileStatus status : fs.globStatus(new Path(baseNamespaceDir, "*"))) {</span> |
| <span class="source-line-no">919</span><span id="line-919"> tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath()));</span> |
| <span class="source-line-no">920</span><span id="line-920"> }</span> |
| <span class="source-line-no">921</span><span id="line-921"> }</span> |
| <span class="source-line-no">922</span><span id="line-922"> return tableDirs;</span> |
| <span class="source-line-no">923</span><span id="line-923"> }</span> |
| <span class="source-line-no">924</span><span id="line-924"></span> |
| <span class="source-line-no">925</span><span id="line-925"> /**</span> |
| <span class="source-line-no">926</span><span id="line-926"> * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders</span> |
| <span class="source-line-no">927</span><span id="line-927"> * such as .logs, .oldlogs, .corrupt folders.</span> |
| <span class="source-line-no">928</span><span id="line-928"> */</span> |
| <span class="source-line-no">929</span><span id="line-929"> public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir)</span> |
| <span class="source-line-no">930</span><span id="line-930"> throws IOException {</span> |
| <span class="source-line-no">931</span><span id="line-931"> // presumes any directory under hbase.rootdir is a table</span> |
| <span class="source-line-no">932</span><span id="line-932"> FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs));</span> |
| <span class="source-line-no">933</span><span id="line-933"> List<Path> tabledirs = new ArrayList<>(dirs.length);</span> |
| <span class="source-line-no">934</span><span id="line-934"> for (FileStatus dir : dirs) {</span> |
| <span class="source-line-no">935</span><span id="line-935"> tabledirs.add(dir.getPath());</span> |
| <span class="source-line-no">936</span><span id="line-936"> }</span> |
| <span class="source-line-no">937</span><span id="line-937"> return tabledirs;</span> |
| <span class="source-line-no">938</span><span id="line-938"> }</span> |
| <span class="source-line-no">939</span><span id="line-939"></span> |
| <span class="source-line-no">940</span><span id="line-940"> /**</span> |
| <span class="source-line-no">941</span><span id="line-941"> * Filter for all dirs that don't start with '.'</span> |
| <span class="source-line-no">942</span><span id="line-942"> */</span> |
| <span class="source-line-no">943</span><span id="line-943"> public static class RegionDirFilter extends AbstractFileStatusFilter {</span> |
| <span class="source-line-no">944</span><span id="line-944"> // This pattern will accept 0.90+ style hex region dirs and older numeric region dir names.</span> |
| <span class="source-line-no">945</span><span id="line-945"> final public static Pattern regionDirPattern = Pattern.compile("^[0-9a-f]*$");</span> |
| <span class="source-line-no">946</span><span id="line-946"> final FileSystem fs;</span> |
| <span class="source-line-no">947</span><span id="line-947"></span> |
| <span class="source-line-no">948</span><span id="line-948"> public RegionDirFilter(FileSystem fs) {</span> |
| <span class="source-line-no">949</span><span id="line-949"> this.fs = fs;</span> |
| <span class="source-line-no">950</span><span id="line-950"> }</span> |
| <span class="source-line-no">951</span><span id="line-951"></span> |
| <span class="source-line-no">952</span><span id="line-952"> @Override</span> |
| <span class="source-line-no">953</span><span id="line-953"> protected boolean accept(Path p, @CheckForNull Boolean isDir) {</span> |
| <span class="source-line-no">954</span><span id="line-954"> if (!regionDirPattern.matcher(p.getName()).matches()) {</span> |
| <span class="source-line-no">955</span><span id="line-955"> return false;</span> |
| <span class="source-line-no">956</span><span id="line-956"> }</span> |
| <span class="source-line-no">957</span><span id="line-957"></span> |
| <span class="source-line-no">958</span><span id="line-958"> try {</span> |
| <span class="source-line-no">959</span><span id="line-959"> return isDirectory(fs, isDir, p);</span> |
| <span class="source-line-no">960</span><span id="line-960"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">961</span><span id="line-961"> // Maybe the file was moved or the fs was disconnected.</span> |
| <span class="source-line-no">962</span><span id="line-962"> LOG.warn("Skipping file {} due to IOException", p, ioe);</span> |
| <span class="source-line-no">963</span><span id="line-963"> return false;</span> |
| <span class="source-line-no">964</span><span id="line-964"> }</span> |
| <span class="source-line-no">965</span><span id="line-965"> }</span> |
| <span class="source-line-no">966</span><span id="line-966"> }</span> |
| <span class="source-line-no">967</span><span id="line-967"></span> |
| <span class="source-line-no">968</span><span id="line-968"> /**</span> |
| <span class="source-line-no">969</span><span id="line-969"> * Given a particular table dir, return all the regiondirs inside it, excluding files such as</span> |
| <span class="source-line-no">970</span><span id="line-970"> * .tableinfo</span> |
| <span class="source-line-no">971</span><span id="line-971"> * @param fs A file system for the Path</span> |
| <span class="source-line-no">972</span><span id="line-972"> * @param tableDir Path to a specific table directory &lt;hbase.rootdir&gt;/&lt;tabledir&gt;</span> |
| <span class="source-line-no">973</span><span id="line-973"> * @return List of paths to valid region directories in table dir.</span> |
| <span class="source-line-no">974</span><span id="line-974"> */</span> |
| <span class="source-line-no">975</span><span id="line-975"> public static List<Path> getRegionDirs(final FileSystem fs, final Path tableDir)</span> |
| <span class="source-line-no">976</span><span id="line-976"> throws IOException {</span> |
| <span class="source-line-no">977</span><span id="line-977"> // assumes we are in a table dir.</span> |
| <span class="source-line-no">978</span><span id="line-978"> List<FileStatus> rds = listStatusWithStatusFilter(fs, tableDir, new RegionDirFilter(fs));</span> |
| <span class="source-line-no">979</span><span id="line-979"> if (rds == null) {</span> |
| <span class="source-line-no">980</span><span id="line-980"> return Collections.emptyList();</span> |
| <span class="source-line-no">981</span><span id="line-981"> }</span> |
| <span class="source-line-no">982</span><span id="line-982"> List<Path> regionDirs = new ArrayList<>(rds.size());</span> |
| <span class="source-line-no">983</span><span id="line-983"> for (FileStatus rdfs : rds) {</span> |
| <span class="source-line-no">984</span><span id="line-984"> Path rdPath = rdfs.getPath();</span> |
| <span class="source-line-no">985</span><span id="line-985"> regionDirs.add(rdPath);</span> |
| <span class="source-line-no">986</span><span id="line-986"> }</span> |
| <span class="source-line-no">987</span><span id="line-987"> return regionDirs;</span> |
| <span class="source-line-no">988</span><span id="line-988"> }</span> |
| <span class="source-line-no">989</span><span id="line-989"></span> |
| <span class="source-line-no">990</span><span id="line-990"> public static Path getRegionDirFromRootDir(Path rootDir, RegionInfo region) {</span> |
| <span class="source-line-no">991</span><span id="line-991"> return getRegionDirFromTableDir(CommonFSUtils.getTableDir(rootDir, region.getTable()), region);</span> |
| <span class="source-line-no">992</span><span id="line-992"> }</span> |
| <span class="source-line-no">993</span><span id="line-993"></span> |
| <span class="source-line-no">994</span><span id="line-994"> public static Path getRegionDirFromTableDir(Path tableDir, RegionInfo region) {</span> |
| <span class="source-line-no">995</span><span id="line-995"> return getRegionDirFromTableDir(tableDir,</span> |
| <span class="source-line-no">996</span><span id="line-996"> ServerRegionReplicaUtil.getRegionInfoForFs(region).getEncodedName());</span> |
| <span class="source-line-no">997</span><span id="line-997"> }</span> |
| <span class="source-line-no">998</span><span id="line-998"></span> |
| <span class="source-line-no">999</span><span id="line-999"> public static Path getRegionDirFromTableDir(Path tableDir, String encodedRegionName) {</span> |
| <span class="source-line-no">1000</span><span id="line-1000"> return new Path(tableDir, encodedRegionName);</span> |
| <span class="source-line-no">1001</span><span id="line-1001"> }</span> |
| <span class="source-line-no">1002</span><span id="line-1002"></span> |
| <span class="source-line-no">1003</span><span id="line-1003"> /**</span> |
| <span class="source-line-no">1004</span><span id="line-1004"> * Filter for all dirs that are legal column family names. This is generally used for colfam dirs</span> |
| <span class="source-line-no">1005</span><span id="line-1005"> * &lt;hbase.rootdir&gt;/&lt;tabledir&gt;/&lt;regiondir&gt;/&lt;colfamdir&gt;.</span> |
| <span class="source-line-no">1006</span><span id="line-1006"> */</span> |
| <span class="source-line-no">1007</span><span id="line-1007"> public static class FamilyDirFilter extends AbstractFileStatusFilter {</span> |
| <span class="source-line-no">1008</span><span id="line-1008"> final FileSystem fs;</span> |
| <span class="source-line-no">1009</span><span id="line-1009"></span> |
| <span class="source-line-no">1010</span><span id="line-1010"> public FamilyDirFilter(FileSystem fs) {</span> |
| <span class="source-line-no">1011</span><span id="line-1011"> this.fs = fs;</span> |
| <span class="source-line-no">1012</span><span id="line-1012"> }</span> |
| <span class="source-line-no">1013</span><span id="line-1013"></span> |
| <span class="source-line-no">1014</span><span id="line-1014"> @Override</span> |
| <span class="source-line-no">1015</span><span id="line-1015"> protected boolean accept(Path p, @CheckForNull Boolean isDir) {</span> |
| <span class="source-line-no">1016</span><span id="line-1016"> try {</span> |
| <span class="source-line-no">1017</span><span id="line-1017"> // throws IAE if invalid</span> |
| <span class="source-line-no">1018</span><span id="line-1018"> ColumnFamilyDescriptorBuilder.isLegalColumnFamilyName(Bytes.toBytes(p.getName()));</span> |
| <span class="source-line-no">1019</span><span id="line-1019"> } catch (IllegalArgumentException iae) {</span> |
| <span class="source-line-no">1020</span><span id="line-1020"> // path name is an invalid family name and thus is excluded.</span> |
| <span class="source-line-no">1021</span><span id="line-1021"> return false;</span> |
| <span class="source-line-no">1022</span><span id="line-1022"> }</span> |
| <span class="source-line-no">1023</span><span id="line-1023"></span> |
| <span class="source-line-no">1024</span><span id="line-1024"> try {</span> |
| <span class="source-line-no">1025</span><span id="line-1025"> return isDirectory(fs, isDir, p);</span> |
| <span class="source-line-no">1026</span><span id="line-1026"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">1027</span><span id="line-1027"> // Maybe the file was moved or the fs was disconnected.</span> |
| <span class="source-line-no">1028</span><span id="line-1028"> LOG.warn("Skipping file {} due to IOException", p, ioe);</span> |
| <span class="source-line-no">1029</span><span id="line-1029"> return false;</span> |
| <span class="source-line-no">1030</span><span id="line-1030"> }</span> |
| <span class="source-line-no">1031</span><span id="line-1031"> }</span> |
| <span class="source-line-no">1032</span><span id="line-1032"> }</span> |
| <span class="source-line-no">1033</span><span id="line-1033"></span> |
| <span class="source-line-no">1034</span><span id="line-1034"> /**</span> |
| <span class="source-line-no">1035</span><span id="line-1035"> * Given a particular region dir, return all the familydirs inside it</span> |
| <span class="source-line-no">1036</span><span id="line-1036"> * @param fs A file system for the Path</span> |
| <span class="source-line-no">1037</span><span id="line-1037"> * @param regionDir Path to a specific region directory</span> |
| <span class="source-line-no">1038</span><span id="line-1038"> * @return List of paths to valid family directories in region dir.</span> |
| <span class="source-line-no">1039</span><span id="line-1039"> */</span> |
| <span class="source-line-no">1040</span><span id="line-1040"> public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir)</span> |
| <span class="source-line-no">1041</span><span id="line-1041"> throws IOException {</span> |
| <span class="source-line-no">1042</span><span id="line-1042"> // assumes we are in a region dir.</span> |
| <span class="source-line-no">1043</span><span id="line-1043"> return getFilePaths(fs, regionDir, new FamilyDirFilter(fs));</span> |
| <span class="source-line-no">1044</span><span id="line-1044"> }</span> |
| <span class="source-line-no">1045</span><span id="line-1045"></span> |
| <span class="source-line-no">1046</span><span id="line-1046"> public static List<Path> getReferenceFilePaths(final FileSystem fs, final Path familyDir)</span> |
| <span class="source-line-no">1047</span><span id="line-1047"> throws IOException {</span> |
| <span class="source-line-no">1048</span><span id="line-1048"> return getFilePaths(fs, familyDir, new ReferenceFileFilter(fs));</span> |
| <span class="source-line-no">1049</span><span id="line-1049"> }</span> |
| <span class="source-line-no">1050</span><span id="line-1050"></span> |
| <span class="source-line-no">1051</span><span id="line-1051"> public static List<Path> getReferenceAndLinkFilePaths(final FileSystem fs, final Path familyDir)</span> |
| <span class="source-line-no">1052</span><span id="line-1052"> throws IOException {</span> |
| <span class="source-line-no">1053</span><span id="line-1053"> return getFilePaths(fs, familyDir, new ReferenceAndLinkFileFilter(fs));</span> |
| <span class="source-line-no">1054</span><span id="line-1054"> }</span> |
| <span class="source-line-no">1055</span><span id="line-1055"></span> |
| <span class="source-line-no">1056</span><span id="line-1056"> private static List<Path> getFilePaths(final FileSystem fs, final Path dir,</span> |
| <span class="source-line-no">1057</span><span id="line-1057"> final PathFilter pathFilter) throws IOException {</span> |
| <span class="source-line-no">1058</span><span id="line-1058"> FileStatus[] fds = fs.listStatus(dir, pathFilter);</span> |
| <span class="source-line-no">1059</span><span id="line-1059"> List<Path> files = new ArrayList<>(fds.length);</span> |
| <span class="source-line-no">1060</span><span id="line-1060"> for (FileStatus fdfs : fds) {</span> |
| <span class="source-line-no">1061</span><span id="line-1061"> Path fdPath = fdfs.getPath();</span> |
| <span class="source-line-no">1062</span><span id="line-1062"> files.add(fdPath);</span> |
| <span class="source-line-no">1063</span><span id="line-1063"> }</span> |
| <span class="source-line-no">1064</span><span id="line-1064"> return files;</span> |
| <span class="source-line-no">1065</span><span id="line-1065"> }</span> |
| <span class="source-line-no">1066</span><span id="line-1066"></span> |
| <span class="source-line-no">1067</span><span id="line-1067"> public static int getRegionReferenceAndLinkFileCount(final FileSystem fs, final Path p) {</span> |
| <span class="source-line-no">1068</span><span id="line-1068"> int result = 0;</span> |
| <span class="source-line-no">1069</span><span id="line-1069"> try {</span> |
| <span class="source-line-no">1070</span><span id="line-1070"> for (Path familyDir : getFamilyDirs(fs, p)) {</span> |
| <span class="source-line-no">1071</span><span id="line-1071"> result += getReferenceAndLinkFilePaths(fs, familyDir).size();</span> |
| <span class="source-line-no">1072</span><span id="line-1072"> }</span> |
| <span class="source-line-no">1073</span><span id="line-1073"> } catch (IOException e) {</span> |
| <span class="source-line-no">1074</span><span id="line-1074"> LOG.warn("Error Counting reference files.", e);</span> |
| <span class="source-line-no">1075</span><span id="line-1075"> }</span> |
| <span class="source-line-no">1076</span><span id="line-1076"> return result;</span> |
| <span class="source-line-no">1077</span><span id="line-1077"> }</span> |
| <span class="source-line-no">1078</span><span id="line-1078"></span> |
| <span class="source-line-no">1079</span><span id="line-1079"> public static class ReferenceAndLinkFileFilter implements PathFilter {</span> |
| <span class="source-line-no">1080</span><span id="line-1080"></span> |
| <span class="source-line-no">1081</span><span id="line-1081"> private final FileSystem fs;</span> |
| <span class="source-line-no">1082</span><span id="line-1082"></span> |
| <span class="source-line-no">1083</span><span id="line-1083"> public ReferenceAndLinkFileFilter(FileSystem fs) {</span> |
| <span class="source-line-no">1084</span><span id="line-1084"> this.fs = fs;</span> |
| <span class="source-line-no">1085</span><span id="line-1085"> }</span> |
| <span class="source-line-no">1086</span><span id="line-1086"></span> |
| <span class="source-line-no">1087</span><span id="line-1087"> @Override</span> |
| <span class="source-line-no">1088</span><span id="line-1088"> public boolean accept(Path rd) {</span> |
| <span class="source-line-no">1089</span><span id="line-1089"> try {</span> |
| <span class="source-line-no">1090</span><span id="line-1090"> // only files can be references.</span> |
| <span class="source-line-no">1091</span><span id="line-1091"> return !fs.getFileStatus(rd).isDirectory()</span> |
| <span class="source-line-no">1092</span><span id="line-1092"> && (StoreFileInfo.isReference(rd) || HFileLink.isHFileLink(rd));</span> |
| <span class="source-line-no">1093</span><span id="line-1093"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">1094</span><span id="line-1094"> // Maybe the file was moved or the fs was disconnected.</span> |
| <span class="source-line-no">1095</span><span id="line-1095"> LOG.warn("Skipping file " + rd + " due to IOException", ioe);</span> |
| <span class="source-line-no">1096</span><span id="line-1096"> return false;</span> |
| <span class="source-line-no">1097</span><span id="line-1097"> }</span> |
| <span class="source-line-no">1098</span><span id="line-1098"> }</span> |
| <span class="source-line-no">1099</span><span id="line-1099"> }</span> |
| <span class="source-line-no">1100</span><span id="line-1100"></span> |
| <span class="source-line-no">1101</span><span id="line-1101"> /**</span> |
| <span class="source-line-no">1102</span><span id="line-1102"> * Filter for HFiles that excludes reference files.</span> |
| <span class="source-line-no">1103</span><span id="line-1103"> */</span> |
| <span class="source-line-no">1104</span><span id="line-1104"> public static class HFileFilter extends AbstractFileStatusFilter {</span> |
| <span class="source-line-no">1105</span><span id="line-1105"> final FileSystem fs;</span> |
| <span class="source-line-no">1106</span><span id="line-1106"></span> |
| <span class="source-line-no">1107</span><span id="line-1107"> public HFileFilter(FileSystem fs) {</span> |
| <span class="source-line-no">1108</span><span id="line-1108"> this.fs = fs;</span> |
| <span class="source-line-no">1109</span><span id="line-1109"> }</span> |
| <span class="source-line-no">1110</span><span id="line-1110"></span> |
| <span class="source-line-no">1111</span><span id="line-1111"> @Override</span> |
| <span class="source-line-no">1112</span><span id="line-1112"> protected boolean accept(Path p, @CheckForNull Boolean isDir) {</span> |
| <span class="source-line-no">1113</span><span id="line-1113"> if (!StoreFileInfo.isHFile(p) && !StoreFileInfo.isMobFile(p)) {</span> |
| <span class="source-line-no">1114</span><span id="line-1114"> return false;</span> |
| <span class="source-line-no">1115</span><span id="line-1115"> }</span> |
| <span class="source-line-no">1116</span><span id="line-1116"></span> |
| <span class="source-line-no">1117</span><span id="line-1117"> try {</span> |
| <span class="source-line-no">1118</span><span id="line-1118"> return isFile(fs, isDir, p);</span> |
| <span class="source-line-no">1119</span><span id="line-1119"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">1120</span><span id="line-1120"> // Maybe the file was moved or the fs was disconnected.</span> |
| <span class="source-line-no">1121</span><span id="line-1121"> LOG.warn("Skipping file {} due to IOException", p, ioe);</span> |
| <span class="source-line-no">1122</span><span id="line-1122"> return false;</span> |
| <span class="source-line-no">1123</span><span id="line-1123"> }</span> |
| <span class="source-line-no">1124</span><span id="line-1124"> }</span> |
| <span class="source-line-no">1125</span><span id="line-1125"> }</span> |
| <span class="source-line-no">1126</span><span id="line-1126"></span> |
| <span class="source-line-no">1127</span><span id="line-1127"> /**</span> |
| <span class="source-line-no">1128</span><span id="line-1128"> * Filter for HFileLinks (StoreFiles and HFiles not included). the filter itself does not consider</span> |
| <span class="source-line-no">1129</span><span id="line-1129"> * if a link is file or not.</span> |
| <span class="source-line-no">1130</span><span id="line-1130"> */</span> |
| <span class="source-line-no">1131</span><span id="line-1131"> public static class HFileLinkFilter implements PathFilter {</span> |
| <span class="source-line-no">1132</span><span id="line-1132"></span> |
| <span class="source-line-no">1133</span><span id="line-1133"> @Override</span> |
| <span class="source-line-no">1134</span><span id="line-1134"> public boolean accept(Path p) {</span> |
| <span class="source-line-no">1135</span><span id="line-1135"> return HFileLink.isHFileLink(p);</span> |
| <span class="source-line-no">1136</span><span id="line-1136"> }</span> |
| <span class="source-line-no">1137</span><span id="line-1137"> }</span> |
| <span class="source-line-no">1138</span><span id="line-1138"></span> |
| <span class="source-line-no">1139</span><span id="line-1139"> public static class ReferenceFileFilter extends AbstractFileStatusFilter {</span> |
| <span class="source-line-no">1140</span><span id="line-1140"></span> |
| <span class="source-line-no">1141</span><span id="line-1141"> private final FileSystem fs;</span> |
| <span class="source-line-no">1142</span><span id="line-1142"></span> |
| <span class="source-line-no">1143</span><span id="line-1143"> public ReferenceFileFilter(FileSystem fs) {</span> |
| <span class="source-line-no">1144</span><span id="line-1144"> this.fs = fs;</span> |
| <span class="source-line-no">1145</span><span id="line-1145"> }</span> |
| <span class="source-line-no">1146</span><span id="line-1146"></span> |
| <span class="source-line-no">1147</span><span id="line-1147"> @Override</span> |
| <span class="source-line-no">1148</span><span id="line-1148"> protected boolean accept(Path p, @CheckForNull Boolean isDir) {</span> |
| <span class="source-line-no">1149</span><span id="line-1149"> if (!StoreFileInfo.isReference(p)) {</span> |
| <span class="source-line-no">1150</span><span id="line-1150"> return false;</span> |
| <span class="source-line-no">1151</span><span id="line-1151"> }</span> |
| <span class="source-line-no">1152</span><span id="line-1152"></span> |
| <span class="source-line-no">1153</span><span id="line-1153"> try {</span> |
| <span class="source-line-no">1154</span><span id="line-1154"> // only files can be references.</span> |
| <span class="source-line-no">1155</span><span id="line-1155"> return isFile(fs, isDir, p);</span> |
| <span class="source-line-no">1156</span><span id="line-1156"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">1157</span><span id="line-1157"> // Maybe the file was moved or the fs was disconnected.</span> |
| <span class="source-line-no">1158</span><span id="line-1158"> LOG.warn("Skipping file {} due to IOException", p, ioe);</span> |
| <span class="source-line-no">1159</span><span id="line-1159"> return false;</span> |
| <span class="source-line-no">1160</span><span id="line-1160"> }</span> |
| <span class="source-line-no">1161</span><span id="line-1161"> }</span> |
| <span class="source-line-no">1162</span><span id="line-1162"> }</span> |
| <span class="source-line-no">1163</span><span id="line-1163"></span> |
| <span class="source-line-no">1164</span><span id="line-1164"> /**</span> |
| <span class="source-line-no">1165</span><span id="line-1165"> * Called every so-often by storefile map builder getTableStoreFilePathMap to report progress.</span> |
| <span class="source-line-no">1166</span><span id="line-1166"> */</span> |
| <span class="source-line-no">1167</span><span id="line-1167"> interface ProgressReporter {</span> |
| <span class="source-line-no">1168</span><span id="line-1168"> /**</span> |
| <span class="source-line-no">1169</span><span id="line-1169"> * @param status File or directory we are about to process.</span> |
| <span class="source-line-no">1170</span><span id="line-1170"> */</span> |
| <span class="source-line-no">1171</span><span id="line-1171"> void progress(FileStatus status);</span> |
| <span class="source-line-no">1172</span><span id="line-1172"> }</span> |
| <span class="source-line-no">1173</span><span id="line-1173"></span> |
| <span class="source-line-no">1174</span><span id="line-1174"> /**</span> |
| <span class="source-line-no">1175</span><span id="line-1175"> * Runs through the HBase rootdir/tablename and creates a reverse lookup map for table StoreFile</span> |
| <span class="source-line-no">1176</span><span id="line-1176"> * names to the full Path. <br></span> |
| <span class="source-line-no">1177</span><span id="line-1177"> * Example...<br></span> |
| <span class="source-line-no">1178</span><span id="line-1178"> * Key = 3944417774205889744 <br></span> |
| <span class="source-line-no">1179</span><span id="line-1179"> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744</span> |
| <span class="source-line-no">1180</span><span id="line-1180"> * @param map map to add values. If null, this method will create and populate one to</span> |
| <span class="source-line-no">1181</span><span id="line-1181"> * return</span> |
| <span class="source-line-no">1182</span><span id="line-1182"> * @param fs The file system to use.</span> |
| <span class="source-line-no">1183</span><span id="line-1183"> * @param hbaseRootDir The root directory to scan.</span> |
| <span class="source-line-no">1184</span><span id="line-1184"> * @param tableName name of the table to scan.</span> |
| <span class="source-line-no">1185</span><span id="line-1185"> * @return Map keyed by StoreFile name with a value of the full Path.</span> |
| <span class="source-line-no">1186</span><span id="line-1186"> * @throws IOException When scanning the directory fails.</span> |
| <span class="source-line-no">1187</span><span id="line-1187"> */</span> |
| <span class="source-line-no">1188</span><span id="line-1188"> public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map,</span> |
| <span class="source-line-no">1189</span><span id="line-1189"> final FileSystem fs, final Path hbaseRootDir, TableName tableName)</span> |
| <span class="source-line-no">1190</span><span id="line-1190"> throws IOException, InterruptedException {</span> |
| <span class="source-line-no">1191</span><span id="line-1191"> return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null, null,</span> |
| <span class="source-line-no">1192</span><span id="line-1192"> (ProgressReporter) null);</span> |
| <span class="source-line-no">1193</span><span id="line-1193"> }</span> |
| <span class="source-line-no">1194</span><span id="line-1194"></span> |
| <span class="source-line-no">1195</span><span id="line-1195"> /**</span> |
| <span class="source-line-no">1196</span><span id="line-1196"> * Runs through the HBase rootdir/tablename and creates a reverse lookup map for table StoreFile</span> |
| <span class="source-line-no">1197</span><span id="line-1197"> * names to the full Path. Note that because this method can be called on a 'live' HBase system</span> |
| <span class="source-line-no">1198</span><span id="line-1198"> * that we will skip files that no longer exist by the time we traverse them and similarly the</span> |
| <span class="source-line-no">1199</span><span id="line-1199"> * user of the result needs to consider that some entries in this map may not exist by the time</span> |
| <span class="source-line-no">1200</span><span id="line-1200"> * this call completes. <br></span> |
| <span class="source-line-no">1201</span><span id="line-1201"> * Example...<br></span> |
| <span class="source-line-no">1202</span><span id="line-1202"> * Key = 3944417774205889744 <br></span> |
| <span class="source-line-no">1203</span><span id="line-1203"> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744</span> |
| <span class="source-line-no">1204</span><span id="line-1204"> * @param resultMap map to add values. If null, this method will create and populate one to</span> |
| <span class="source-line-no">1205</span><span id="line-1205"> * return</span> |
| <span class="source-line-no">1206</span><span id="line-1206"> * @param fs The file system to use.</span> |
| <span class="source-line-no">1207</span><span id="line-1207"> * @param hbaseRootDir The root directory to scan.</span> |
| <span class="source-line-no">1208</span><span id="line-1208"> * @param tableName name of the table to scan.</span> |
| <span class="source-line-no">1209</span><span id="line-1209"> * @param sfFilter optional path filter to apply to store files</span> |
| <span class="source-line-no">1210</span><span id="line-1210"> * @param executor optional executor service to parallelize this operation</span> |
| <span class="source-line-no">1211</span><span id="line-1211"> * @param progressReporter Instance or null; gets called every time we move to new region of</span> |
| <span class="source-line-no">1212</span><span id="line-1212"> * family dir and for each store file.</span> |
| <span class="source-line-no">1213</span><span id="line-1213"> * @return Map keyed by StoreFile name with a value of the full Path.</span> |
| <span class="source-line-no">1214</span><span id="line-1214"> * @throws IOException When scanning the directory fails.</span> |
| <span class="source-line-no">1215</span><span id="line-1215"> * @deprecated Since 2.3.0. For removal in hbase4. Use ProgressReporter override instead.</span> |
| <span class="source-line-no">1216</span><span id="line-1216"> */</span> |
| <span class="source-line-no">1217</span><span id="line-1217"> @Deprecated</span> |
| <span class="source-line-no">1218</span><span id="line-1218"> public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> resultMap,</span> |
| <span class="source-line-no">1219</span><span id="line-1219"> final FileSystem fs, final Path hbaseRootDir, TableName tableName, final PathFilter sfFilter,</span> |
| <span class="source-line-no">1220</span><span id="line-1220"> ExecutorService executor, final HbckErrorReporter progressReporter)</span> |
| <span class="source-line-no">1221</span><span id="line-1221"> throws IOException, InterruptedException {</span> |
| <span class="source-line-no">1222</span><span id="line-1222"> return getTableStoreFilePathMap(resultMap, fs, hbaseRootDir, tableName, sfFilter, executor,</span> |
| <span class="source-line-no">1223</span><span id="line-1223"> new ProgressReporter() {</span> |
| <span class="source-line-no">1224</span><span id="line-1224"> @Override</span> |
| <span class="source-line-no">1225</span><span id="line-1225"> public void progress(FileStatus status) {</span> |
| <span class="source-line-no">1226</span><span id="line-1226"> // status is not used in this implementation.</span> |
| <span class="source-line-no">1227</span><span id="line-1227"> progressReporter.progress();</span> |
| <span class="source-line-no">1228</span><span id="line-1228"> }</span> |
| <span class="source-line-no">1229</span><span id="line-1229"> });</span> |
| <span class="source-line-no">1230</span><span id="line-1230"> }</span> |
| <span class="source-line-no">1231</span><span id="line-1231"></span> |
| <span class="source-line-no">1232</span><span id="line-1232"> /**</span> |
| <span class="source-line-no">1233</span><span id="line-1233"> * Runs through the HBase rootdir/tablename and creates a reverse lookup map for table StoreFile</span> |
| <span class="source-line-no">1234</span><span id="line-1234"> * names to the full Path. Note that because this method can be called on a 'live' HBase system</span> |
| <span class="source-line-no">1235</span><span id="line-1235"> * that we will skip files that no longer exist by the time we traverse them and similarly the</span> |
| <span class="source-line-no">1236</span><span id="line-1236"> * user of the result needs to consider that some entries in this map may not exist by the time</span> |
| <span class="source-line-no">1237</span><span id="line-1237"> * this call completes. <br></span> |
| <span class="source-line-no">1238</span><span id="line-1238"> * Example...<br></span> |
| <span class="source-line-no">1239</span><span id="line-1239"> * Key = 3944417774205889744 <br></span> |
| <span class="source-line-no">1240</span><span id="line-1240"> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744</span> |
| <span class="source-line-no">1241</span><span id="line-1241"> * @param resultMap map to add values. If null, this method will create and populate one to</span> |
| <span class="source-line-no">1242</span><span id="line-1242"> * return</span> |
| <span class="source-line-no">1243</span><span id="line-1243"> * @param fs The file system to use.</span> |
| <span class="source-line-no">1244</span><span id="line-1244"> * @param hbaseRootDir The root directory to scan.</span> |
| <span class="source-line-no">1245</span><span id="line-1245"> * @param tableName name of the table to scan.</span> |
| <span class="source-line-no">1246</span><span id="line-1246"> * @param sfFilter optional path filter to apply to store files</span> |
| <span class="source-line-no">1247</span><span id="line-1247"> * @param executor optional executor service to parallelize this operation</span> |
| <span class="source-line-no">1248</span><span id="line-1248"> * @param progressReporter Instance or null; gets called every time we move to new region of</span> |
| <span class="source-line-no">1249</span><span id="line-1249"> * family dir and for each store file.</span> |
| <span class="source-line-no">1250</span><span id="line-1250"> * @return Map keyed by StoreFile name with a value of the full Path.</span> |
| <span class="source-line-no">1251</span><span id="line-1251"> * @throws IOException When scanning the directory fails.</span> |
| <span class="source-line-no">1252</span><span id="line-1252"> * @throws InterruptedException the thread is interrupted, either before or during the activity.</span> |
| <span class="source-line-no">1253</span><span id="line-1253"> */</span> |
| <span class="source-line-no">1254</span><span id="line-1254"> public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> resultMap,</span> |
| <span class="source-line-no">1255</span><span id="line-1255"> final FileSystem fs, final Path hbaseRootDir, TableName tableName, final PathFilter sfFilter,</span> |
| <span class="source-line-no">1256</span><span id="line-1256"> ExecutorService executor, final ProgressReporter progressReporter)</span> |
| <span class="source-line-no">1257</span><span id="line-1257"> throws IOException, InterruptedException {</span> |
| <span class="source-line-no">1258</span><span id="line-1258"></span> |
| <span class="source-line-no">1259</span><span id="line-1259"> final Map<String, Path> finalResultMap =</span> |
| <span class="source-line-no">1260</span><span id="line-1260"> resultMap == null ? new ConcurrentHashMap<>(128, 0.75f, 32) : resultMap;</span> |
| <span class="source-line-no">1261</span><span id="line-1261"></span> |
| <span class="source-line-no">1262</span><span id="line-1262"> // only include the directory paths to tables</span> |
| <span class="source-line-no">1263</span><span id="line-1263"> Path tableDir = CommonFSUtils.getTableDir(hbaseRootDir, tableName);</span> |
| <span class="source-line-no">1264</span><span id="line-1264"> // Inside a table, there are compaction.dir directories to skip. Otherwise, all else</span> |
| <span class="source-line-no">1265</span><span id="line-1265"> // should be regions.</span> |
| <span class="source-line-no">1266</span><span id="line-1266"> final FamilyDirFilter familyFilter = new FamilyDirFilter(fs);</span> |
| <span class="source-line-no">1267</span><span id="line-1267"> final Vector<Exception> exceptions = new Vector<>();</span> |
| <span class="source-line-no">1268</span><span id="line-1268"></span> |
| <span class="source-line-no">1269</span><span id="line-1269"> try {</span> |
| <span class="source-line-no">1270</span><span id="line-1270"> List<FileStatus> regionDirs =</span> |
| <span class="source-line-no">1271</span><span id="line-1271"> FSUtils.listStatusWithStatusFilter(fs, tableDir, new RegionDirFilter(fs));</span> |
| <span class="source-line-no">1272</span><span id="line-1272"> if (regionDirs == null) {</span> |
| <span class="source-line-no">1273</span><span id="line-1273"> return finalResultMap;</span> |
| <span class="source-line-no">1274</span><span id="line-1274"> }</span> |
| <span class="source-line-no">1275</span><span id="line-1275"></span> |
| <span class="source-line-no">1276</span><span id="line-1276"> final List<Future<?>> futures = new ArrayList<>(regionDirs.size());</span> |
| <span class="source-line-no">1277</span><span id="line-1277"></span> |
| <span class="source-line-no">1278</span><span id="line-1278"> for (FileStatus regionDir : regionDirs) {</span> |
| <span class="source-line-no">1279</span><span id="line-1279"> if (null != progressReporter) {</span> |
| <span class="source-line-no">1280</span><span id="line-1280"> progressReporter.progress(regionDir);</span> |
| <span class="source-line-no">1281</span><span id="line-1281"> }</span> |
| <span class="source-line-no">1282</span><span id="line-1282"> final Path dd = regionDir.getPath();</span> |
| <span class="source-line-no">1283</span><span id="line-1283"></span> |
| <span class="source-line-no">1284</span><span id="line-1284"> if (!exceptions.isEmpty()) {</span> |
| <span class="source-line-no">1285</span><span id="line-1285"> break;</span> |
| <span class="source-line-no">1286</span><span id="line-1286"> }</span> |
| <span class="source-line-no">1287</span><span id="line-1287"></span> |
| <span class="source-line-no">1288</span><span id="line-1288"> Runnable getRegionStoreFileMapCall = new Runnable() {</span> |
| <span class="source-line-no">1289</span><span id="line-1289"> @Override</span> |
| <span class="source-line-no">1290</span><span id="line-1290"> public void run() {</span> |
| <span class="source-line-no">1291</span><span id="line-1291"> try {</span> |
| <span class="source-line-no">1292</span><span id="line-1292"> HashMap<String, Path> regionStoreFileMap = new HashMap<>();</span> |
| <span class="source-line-no">1293</span><span id="line-1293"> List<FileStatus> familyDirs =</span> |
| <span class="source-line-no">1294</span><span id="line-1294"> FSUtils.listStatusWithStatusFilter(fs, dd, familyFilter);</span> |
| <span class="source-line-no">1295</span><span id="line-1295"> if (familyDirs == null) {</span> |
| <span class="source-line-no">1296</span><span id="line-1296"> if (!fs.exists(dd)) {</span> |
| <span class="source-line-no">1297</span><span id="line-1297"> LOG.warn("Skipping region because it no longer exists: " + dd);</span> |
| <span class="source-line-no">1298</span><span id="line-1298"> } else {</span> |
| <span class="source-line-no">1299</span><span id="line-1299"> LOG.warn("Skipping region because it has no family dirs: " + dd);</span> |
| <span class="source-line-no">1300</span><span id="line-1300"> }</span> |
| <span class="source-line-no">1301</span><span id="line-1301"> return;</span> |
| <span class="source-line-no">1302</span><span id="line-1302"> }</span> |
| <span class="source-line-no">1303</span><span id="line-1303"> for (FileStatus familyDir : familyDirs) {</span> |
| <span class="source-line-no">1304</span><span id="line-1304"> if (null != progressReporter) {</span> |
| <span class="source-line-no">1305</span><span id="line-1305"> progressReporter.progress(familyDir);</span> |
| <span class="source-line-no">1306</span><span id="line-1306"> }</span> |
| <span class="source-line-no">1307</span><span id="line-1307"> Path family = familyDir.getPath();</span> |
| <span class="source-line-no">1308</span><span id="line-1308"> if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) {</span> |
| <span class="source-line-no">1309</span><span id="line-1309"> continue;</span> |
| <span class="source-line-no">1310</span><span id="line-1310"> }</span> |
| <span class="source-line-no">1311</span><span id="line-1311"> // now in family, iterate over the StoreFiles and</span> |
| <span class="source-line-no">1312</span><span id="line-1312"> // put in map</span> |
| <span class="source-line-no">1313</span><span id="line-1313"> FileStatus[] familyStatus = fs.listStatus(family);</span> |
| <span class="source-line-no">1314</span><span id="line-1314"> for (FileStatus sfStatus : familyStatus) {</span> |
| <span class="source-line-no">1315</span><span id="line-1315"> if (null != progressReporter) {</span> |
| <span class="source-line-no">1316</span><span id="line-1316"> progressReporter.progress(sfStatus);</span> |
| <span class="source-line-no">1317</span><span id="line-1317"> }</span> |
| <span class="source-line-no">1318</span><span id="line-1318"> Path sf = sfStatus.getPath();</span> |
| <span class="source-line-no">1319</span><span id="line-1319"> if (sfFilter == null || sfFilter.accept(sf)) {</span> |
| <span class="source-line-no">1320</span><span id="line-1320"> regionStoreFileMap.put(sf.getName(), sf);</span> |
| <span class="source-line-no">1321</span><span id="line-1321"> }</span> |
| <span class="source-line-no">1322</span><span id="line-1322"> }</span> |
| <span class="source-line-no">1323</span><span id="line-1323"> }</span> |
| <span class="source-line-no">1324</span><span id="line-1324"> finalResultMap.putAll(regionStoreFileMap);</span> |
| <span class="source-line-no">1325</span><span id="line-1325"> } catch (Exception e) {</span> |
| <span class="source-line-no">1326</span><span id="line-1326"> LOG.error("Could not get region store file map for region: " + dd, e);</span> |
| <span class="source-line-no">1327</span><span id="line-1327"> exceptions.add(e);</span> |
| <span class="source-line-no">1328</span><span id="line-1328"> }</span> |
| <span class="source-line-no">1329</span><span id="line-1329"> }</span> |
| <span class="source-line-no">1330</span><span id="line-1330"> };</span> |
| <span class="source-line-no">1331</span><span id="line-1331"></span> |
| <span class="source-line-no">1332</span><span id="line-1332"> // If executor is available, submit async tasks to exec concurrently, otherwise</span> |
| <span class="source-line-no">1333</span><span id="line-1333"> // just do serial sync execution</span> |
| <span class="source-line-no">1334</span><span id="line-1334"> if (executor != null) {</span> |
| <span class="source-line-no">1335</span><span id="line-1335"> Future<?> future = executor.submit(getRegionStoreFileMapCall);</span> |
| <span class="source-line-no">1336</span><span id="line-1336"> futures.add(future);</span> |
| <span class="source-line-no">1337</span><span id="line-1337"> } else {</span> |
| <span class="source-line-no">1338</span><span id="line-1338"> FutureTask<?> future = new FutureTask<>(getRegionStoreFileMapCall, null);</span> |
| <span class="source-line-no">1339</span><span id="line-1339"> future.run();</span> |
| <span class="source-line-no">1340</span><span id="line-1340"> futures.add(future);</span> |
| <span class="source-line-no">1341</span><span id="line-1341"> }</span> |
| <span class="source-line-no">1342</span><span id="line-1342"> }</span> |
| <span class="source-line-no">1343</span><span id="line-1343"></span> |
| <span class="source-line-no">1344</span><span id="line-1344"> // Ensure all pending tasks are complete (or that we run into an exception)</span> |
| <span class="source-line-no">1345</span><span id="line-1345"> for (Future<?> f : futures) {</span> |
| <span class="source-line-no">1346</span><span id="line-1346"> if (!exceptions.isEmpty()) {</span> |
| <span class="source-line-no">1347</span><span id="line-1347"> break;</span> |
| <span class="source-line-no">1348</span><span id="line-1348"> }</span> |
| <span class="source-line-no">1349</span><span id="line-1349"> try {</span> |
| <span class="source-line-no">1350</span><span id="line-1350"> f.get();</span> |
| <span class="source-line-no">1351</span><span id="line-1351"> } catch (ExecutionException e) {</span> |
| <span class="source-line-no">1352</span><span id="line-1352"> LOG.error("Unexpected exec exception! Should've been caught already. (Bug?)", e);</span> |
| <span class="source-line-no">1353</span><span id="line-1353"> // Shouldn't happen, we already logged/caught any exceptions in the Runnable</span> |
| <span class="source-line-no">1354</span><span id="line-1354"> }</span> |
| <span class="source-line-no">1355</span><span id="line-1355"> }</span> |
| <span class="source-line-no">1356</span><span id="line-1356"> } catch (IOException e) {</span> |
| <span class="source-line-no">1357</span><span id="line-1357"> LOG.error("Cannot execute getTableStoreFilePathMap for " + tableName, e);</span> |
| <span class="source-line-no">1358</span><span id="line-1358"> exceptions.add(e);</span> |
| <span class="source-line-no">1359</span><span id="line-1359"> } finally {</span> |
| <span class="source-line-no">1360</span><span id="line-1360"> if (!exceptions.isEmpty()) {</span> |
| <span class="source-line-no">1361</span><span id="line-1361"> // Just throw the first exception as an indication something bad happened</span> |
| <span class="source-line-no">1362</span><span id="line-1362"> // Don't need to propagate all the exceptions, we already logged them all anyway</span> |
| <span class="source-line-no">1363</span><span id="line-1363"> Throwables.propagateIfPossible(exceptions.firstElement(), IOException.class);</span> |
| <span class="source-line-no">1364</span><span id="line-1364"> throw new IOException(exceptions.firstElement());</span> |
| <span class="source-line-no">1365</span><span id="line-1365"> }</span> |
| <span class="source-line-no">1366</span><span id="line-1366"> }</span> |
| <span class="source-line-no">1367</span><span id="line-1367"></span> |
| <span class="source-line-no">1368</span><span id="line-1368"> return finalResultMap;</span> |
| <span class="source-line-no">1369</span><span id="line-1369"> }</span> |
| <span class="source-line-no">1370</span><span id="line-1370"></span> |
| <span class="source-line-no">1371</span><span id="line-1371"> public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) {</span> |
| <span class="source-line-no">1372</span><span id="line-1372"> int result = 0;</span> |
| <span class="source-line-no">1373</span><span id="line-1373"> try {</span> |
| <span class="source-line-no">1374</span><span id="line-1374"> for (Path familyDir : getFamilyDirs(fs, p)) {</span> |
| <span class="source-line-no">1375</span><span id="line-1375"> result += getReferenceFilePaths(fs, familyDir).size();</span> |
| <span class="source-line-no">1376</span><span id="line-1376"> }</span> |
| <span class="source-line-no">1377</span><span id="line-1377"> } catch (IOException e) {</span> |
| <span class="source-line-no">1378</span><span id="line-1378"> LOG.warn("Error counting reference files", e);</span> |
| <span class="source-line-no">1379</span><span id="line-1379"> }</span> |
| <span class="source-line-no">1380</span><span id="line-1380"> return result;</span> |
| <span class="source-line-no">1381</span><span id="line-1381"> }</span> |
| <span class="source-line-no">1382</span><span id="line-1382"></span> |
| <span class="source-line-no">1383</span><span id="line-1383"> /**</span> |
| <span class="source-line-no">1384</span><span id="line-1384"> * Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to</span> |
| <span class="source-line-no">1385</span><span id="line-1385"> * the full Path. <br></span> |
| <span class="source-line-no">1386</span><span id="line-1386"> * Example...<br></span> |
| <span class="source-line-no">1387</span><span id="line-1387"> * Key = 3944417774205889744 <br></span> |
| <span class="source-line-no">1388</span><span id="line-1388"> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744</span> |
| <span class="source-line-no">1389</span><span id="line-1389"> * @param fs The file system to use.</span> |
| <span class="source-line-no">1390</span><span id="line-1390"> * @param hbaseRootDir The root directory to scan.</span> |
| <span class="source-line-no">1391</span><span id="line-1391"> * @return Map keyed by StoreFile name with a value of the full Path.</span> |
| <span class="source-line-no">1392</span><span id="line-1392"> * @throws IOException When scanning the directory fails.</span> |
| <span class="source-line-no">1393</span><span id="line-1393"> */</span> |
| <span class="source-line-no">1394</span><span id="line-1394"> public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,</span> |
| <span class="source-line-no">1395</span><span id="line-1395"> final Path hbaseRootDir) throws IOException, InterruptedException {</span> |
| <span class="source-line-no">1396</span><span id="line-1396"> return getTableStoreFilePathMap(fs, hbaseRootDir, null, null, (ProgressReporter) null);</span> |
| <span class="source-line-no">1397</span><span id="line-1397"> }</span> |
| <span class="source-line-no">1398</span><span id="line-1398"></span> |
| <span class="source-line-no">1399</span><span id="line-1399"> /**</span> |
| <span class="source-line-no">1400</span><span id="line-1400"> * Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to</span> |
| <span class="source-line-no">1401</span><span id="line-1401"> * the full Path. <br></span> |
| <span class="source-line-no">1402</span><span id="line-1402"> * Example...<br></span> |
| <span class="source-line-no">1403</span><span id="line-1403"> * Key = 3944417774205889744 <br></span> |
| <span class="source-line-no">1404</span><span id="line-1404"> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744</span> |
| <span class="source-line-no">1405</span><span id="line-1405"> * @param fs The file system to use.</span> |
| <span class="source-line-no">1406</span><span id="line-1406"> * @param hbaseRootDir The root directory to scan.</span> |
| <span class="source-line-no">1407</span><span id="line-1407"> * @param sfFilter optional path filter to apply to store files</span> |
| <span class="source-line-no">1408</span><span id="line-1408"> * @param executor optional executor service to parallelize this operation</span> |
| <span class="source-line-no">1409</span><span id="line-1409"> * @param progressReporter Instance or null; gets called every time we move to new region of</span> |
| <span class="source-line-no">1410</span><span id="line-1410"> * family dir and for each store file.</span> |
| <span class="source-line-no">1411</span><span id="line-1411"> * @return Map keyed by StoreFile name with a value of the full Path.</span> |
| <span class="source-line-no">1412</span><span id="line-1412"> * @throws IOException When scanning the directory fails.</span> |
| <span class="source-line-no">1413</span><span id="line-1413"> * @deprecated Since 2.3.0. Will be removed in hbase4. Used</span> |
| <span class="source-line-no">1414</span><span id="line-1414"> * {@link #getTableStoreFilePathMap(FileSystem, Path, PathFilter, ExecutorService, ProgressReporter)}</span> |
| <span class="source-line-no">1415</span><span id="line-1415"> */</span> |
| <span class="source-line-no">1416</span><span id="line-1416"> @Deprecated</span> |
| <span class="source-line-no">1417</span><span id="line-1417"> public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,</span> |
| <span class="source-line-no">1418</span><span id="line-1418"> final Path hbaseRootDir, PathFilter sfFilter, ExecutorService executor,</span> |
| <span class="source-line-no">1419</span><span id="line-1419"> HbckErrorReporter progressReporter) throws IOException, InterruptedException {</span> |
| <span class="source-line-no">1420</span><span id="line-1420"> return getTableStoreFilePathMap(fs, hbaseRootDir, sfFilter, executor, new ProgressReporter() {</span> |
| <span class="source-line-no">1421</span><span id="line-1421"> @Override</span> |
| <span class="source-line-no">1422</span><span id="line-1422"> public void progress(FileStatus status) {</span> |
| <span class="source-line-no">1423</span><span id="line-1423"> // status is not used in this implementation.</span> |
| <span class="source-line-no">1424</span><span id="line-1424"> progressReporter.progress();</span> |
| <span class="source-line-no">1425</span><span id="line-1425"> }</span> |
| <span class="source-line-no">1426</span><span id="line-1426"> });</span> |
| <span class="source-line-no">1427</span><span id="line-1427"> }</span> |
| <span class="source-line-no">1428</span><span id="line-1428"></span> |
| <span class="source-line-no">1429</span><span id="line-1429"> /**</span> |
| <span class="source-line-no">1430</span><span id="line-1430"> * Runs through the HBase rootdir and creates a reverse lookup map for table StoreFile names to</span> |
| <span class="source-line-no">1431</span><span id="line-1431"> * the full Path. <br></span> |
| <span class="source-line-no">1432</span><span id="line-1432"> * Example...<br></span> |
| <span class="source-line-no">1433</span><span id="line-1433"> * Key = 3944417774205889744 <br></span> |
| <span class="source-line-no">1434</span><span id="line-1434"> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744</span> |
| <span class="source-line-no">1435</span><span id="line-1435"> * @param fs The file system to use.</span> |
| <span class="source-line-no">1436</span><span id="line-1436"> * @param hbaseRootDir The root directory to scan.</span> |
| <span class="source-line-no">1437</span><span id="line-1437"> * @param sfFilter optional path filter to apply to store files</span> |
| <span class="source-line-no">1438</span><span id="line-1438"> * @param executor optional executor service to parallelize this operation</span> |
| <span class="source-line-no">1439</span><span id="line-1439"> * @param progressReporter Instance or null; gets called every time we move to new region of</span> |
| <span class="source-line-no">1440</span><span id="line-1440"> * family dir and for each store file.</span> |
| <span class="source-line-no">1441</span><span id="line-1441"> * @return Map keyed by StoreFile name with a value of the full Path.</span> |
| <span class="source-line-no">1442</span><span id="line-1442"> * @throws IOException When scanning the directory fails.</span> |
| <span class="source-line-no">1443</span><span id="line-1443"> */</span> |
| <span class="source-line-no">1444</span><span id="line-1444"> public static Map<String, Path> getTableStoreFilePathMap(final FileSystem fs,</span> |
| <span class="source-line-no">1445</span><span id="line-1445"> final Path hbaseRootDir, PathFilter sfFilter, ExecutorService executor,</span> |
| <span class="source-line-no">1446</span><span id="line-1446"> ProgressReporter progressReporter) throws IOException, InterruptedException {</span> |
| <span class="source-line-no">1447</span><span id="line-1447"> ConcurrentHashMap<String, Path> map = new ConcurrentHashMap<>(1024, 0.75f, 32);</span> |
| <span class="source-line-no">1448</span><span id="line-1448"></span> |
| <span class="source-line-no">1449</span><span id="line-1449"> // if this method looks similar to 'getTableFragmentation' that is because</span> |
| <span class="source-line-no">1450</span><span id="line-1450"> // it was borrowed from it.</span> |
| <span class="source-line-no">1451</span><span id="line-1451"></span> |
| <span class="source-line-no">1452</span><span id="line-1452"> // only include the directory paths to tables</span> |
| <span class="source-line-no">1453</span><span id="line-1453"> for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) {</span> |
| <span class="source-line-no">1454</span><span id="line-1454"> getTableStoreFilePathMap(map, fs, hbaseRootDir, CommonFSUtils.getTableName(tableDir),</span> |
| <span class="source-line-no">1455</span><span id="line-1455"> sfFilter, executor, progressReporter);</span> |
| <span class="source-line-no">1456</span><span id="line-1456"> }</span> |
| <span class="source-line-no">1457</span><span id="line-1457"> return map;</span> |
| <span class="source-line-no">1458</span><span id="line-1458"> }</span> |
| <span class="source-line-no">1459</span><span id="line-1459"></span> |
| <span class="source-line-no">1460</span><span id="line-1460"> /**</span> |
| <span class="source-line-no">1461</span><span id="line-1461"> * Filters FileStatuses in an array and returns a list</span> |
| <span class="source-line-no">1462</span><span id="line-1462"> * @param input An array of FileStatuses</span> |
| <span class="source-line-no">1463</span><span id="line-1463"> * @param filter A required filter to filter the array</span> |
| <span class="source-line-no">1464</span><span id="line-1464"> * @return A list of FileStatuses</span> |
| <span class="source-line-no">1465</span><span id="line-1465"> */</span> |
| <span class="source-line-no">1466</span><span id="line-1466"> public static List<FileStatus> filterFileStatuses(FileStatus[] input, FileStatusFilter filter) {</span> |
| <span class="source-line-no">1467</span><span id="line-1467"> if (input == null) return null;</span> |
| <span class="source-line-no">1468</span><span id="line-1468"> return filterFileStatuses(Iterators.forArray(input), filter);</span> |
| <span class="source-line-no">1469</span><span id="line-1469"> }</span> |
| <span class="source-line-no">1470</span><span id="line-1470"></span> |
| <span class="source-line-no">1471</span><span id="line-1471"> /**</span> |
| <span class="source-line-no">1472</span><span id="line-1472"> * Filters FileStatuses in an iterator and returns a list</span> |
| <span class="source-line-no">1473</span><span id="line-1473"> * @param input An iterator of FileStatuses</span> |
| <span class="source-line-no">1474</span><span id="line-1474"> * @param filter A required filter to filter the array</span> |
| <span class="source-line-no">1475</span><span id="line-1475"> * @return A list of FileStatuses</span> |
| <span class="source-line-no">1476</span><span id="line-1476"> */</span> |
| <span class="source-line-no">1477</span><span id="line-1477"> public static List<FileStatus> filterFileStatuses(Iterator<FileStatus> input,</span> |
| <span class="source-line-no">1478</span><span id="line-1478"> FileStatusFilter filter) {</span> |
| <span class="source-line-no">1479</span><span id="line-1479"> if (input == null) return null;</span> |
| <span class="source-line-no">1480</span><span id="line-1480"> ArrayList<FileStatus> results = new ArrayList<>();</span> |
| <span class="source-line-no">1481</span><span id="line-1481"> while (input.hasNext()) {</span> |
| <span class="source-line-no">1482</span><span id="line-1482"> FileStatus f = input.next();</span> |
| <span class="source-line-no">1483</span><span id="line-1483"> if (filter.accept(f)) {</span> |
| <span class="source-line-no">1484</span><span id="line-1484"> results.add(f);</span> |
| <span class="source-line-no">1485</span><span id="line-1485"> }</span> |
| <span class="source-line-no">1486</span><span id="line-1486"> }</span> |
| <span class="source-line-no">1487</span><span id="line-1487"> return results;</span> |
| <span class="source-line-no">1488</span><span id="line-1488"> }</span> |
| <span class="source-line-no">1489</span><span id="line-1489"></span> |
| <span class="source-line-no">1490</span><span id="line-1490"> /**</span> |
| <span class="source-line-no">1491</span><span id="line-1491"> * Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates</span> |
| <span class="source-line-no">1492</span><span id="line-1492"> * differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and</span> |
| <span class="source-line-no">1493</span><span id="line-1493"> * return an empty FileStatus[] while Hadoop 2 will throw FileNotFoundException.</span> |
| <span class="source-line-no">1494</span><span id="line-1494"> * @param fs file system</span> |
| <span class="source-line-no">1495</span><span id="line-1495"> * @param dir directory</span> |
| <span class="source-line-no">1496</span><span id="line-1496"> * @param filter file status filter</span> |
| <span class="source-line-no">1497</span><span id="line-1497"> * @return null if dir is empty or doesn't exist, otherwise FileStatus list</span> |
| <span class="source-line-no">1498</span><span id="line-1498"> */</span> |
| <span class="source-line-no">1499</span><span id="line-1499"> public static List<FileStatus> listStatusWithStatusFilter(final FileSystem fs, final Path dir,</span> |
| <span class="source-line-no">1500</span><span id="line-1500"> final FileStatusFilter filter) throws IOException {</span> |
| <span class="source-line-no">1501</span><span id="line-1501"> FileStatus[] status = null;</span> |
| <span class="source-line-no">1502</span><span id="line-1502"> try {</span> |
| <span class="source-line-no">1503</span><span id="line-1503"> status = fs.listStatus(dir);</span> |
| <span class="source-line-no">1504</span><span id="line-1504"> } catch (FileNotFoundException fnfe) {</span> |
| <span class="source-line-no">1505</span><span id="line-1505"> LOG.trace("{} does not exist", dir);</span> |
| <span class="source-line-no">1506</span><span id="line-1506"> return null;</span> |
| <span class="source-line-no">1507</span><span id="line-1507"> }</span> |
| <span class="source-line-no">1508</span><span id="line-1508"></span> |
| <span class="source-line-no">1509</span><span id="line-1509"> if (ArrayUtils.getLength(status) == 0) {</span> |
| <span class="source-line-no">1510</span><span id="line-1510"> return null;</span> |
| <span class="source-line-no">1511</span><span id="line-1511"> }</span> |
| <span class="source-line-no">1512</span><span id="line-1512"></span> |
| <span class="source-line-no">1513</span><span id="line-1513"> if (filter == null) {</span> |
| <span class="source-line-no">1514</span><span id="line-1514"> return Arrays.asList(status);</span> |
| <span class="source-line-no">1515</span><span id="line-1515"> } else {</span> |
| <span class="source-line-no">1516</span><span id="line-1516"> List<FileStatus> status2 = filterFileStatuses(status, filter);</span> |
| <span class="source-line-no">1517</span><span id="line-1517"> if (status2 == null || status2.isEmpty()) {</span> |
| <span class="source-line-no">1518</span><span id="line-1518"> return null;</span> |
| <span class="source-line-no">1519</span><span id="line-1519"> } else {</span> |
| <span class="source-line-no">1520</span><span id="line-1520"> return status2;</span> |
| <span class="source-line-no">1521</span><span id="line-1521"> }</span> |
| <span class="source-line-no">1522</span><span id="line-1522"> }</span> |
| <span class="source-line-no">1523</span><span id="line-1523"> }</span> |
| <span class="source-line-no">1524</span><span id="line-1524"></span> |
| <span class="source-line-no">1525</span><span id="line-1525"> /**</span> |
| <span class="source-line-no">1526</span><span id="line-1526"> * This function is to scan the root path of the file system to get the degree of locality for</span> |
| <span class="source-line-no">1527</span><span id="line-1527"> * each region on each of the servers having at least one block of that region. This is used by</span> |
| <span class="source-line-no">1528</span><span id="line-1528"> * the tool {@link org.apache.hadoop.hbase.master.RegionPlacementMaintainer} the configuration to</span> |
| <span class="source-line-no">1529</span><span id="line-1529"> * use</span> |
| <span class="source-line-no">1530</span><span id="line-1530"> * @return the mapping from region encoded name to a map of server names to locality fraction in</span> |
| <span class="source-line-no">1531</span><span id="line-1531"> * case of file system errors or interrupts</span> |
| <span class="source-line-no">1532</span><span id="line-1532"> */</span> |
| <span class="source-line-no">1533</span><span id="line-1533"> public static Map<String, Map<String, Float>></span> |
| <span class="source-line-no">1534</span><span id="line-1534"> getRegionDegreeLocalityMappingFromFS(final Configuration conf) throws IOException {</span> |
| <span class="source-line-no">1535</span><span id="line-1535"> return getRegionDegreeLocalityMappingFromFS(conf, null,</span> |
| <span class="source-line-no">1536</span><span id="line-1536"> conf.getInt(THREAD_POOLSIZE, DEFAULT_THREAD_POOLSIZE));</span> |
| <span class="source-line-no">1537</span><span id="line-1537"></span> |
| <span class="source-line-no">1538</span><span id="line-1538"> }</span> |
| <span class="source-line-no">1539</span><span id="line-1539"></span> |
| <span class="source-line-no">1540</span><span id="line-1540"> /**</span> |
| <span class="source-line-no">1541</span><span id="line-1541"> * This function is to scan the root path of the file system to get the degree of locality for</span> |
| <span class="source-line-no">1542</span><span id="line-1542"> * each region on each of the servers having at least one block of that region. the configuration</span> |
| <span class="source-line-no">1543</span><span id="line-1543"> * to use the table you wish to scan locality for the thread pool size to use</span> |
| <span class="source-line-no">1544</span><span id="line-1544"> * @return the mapping from region encoded name to a map of server names to locality fraction in</span> |
| <span class="source-line-no">1545</span><span id="line-1545"> * case of file system errors or interrupts</span> |
| <span class="source-line-no">1546</span><span id="line-1546"> */</span> |
| <span class="source-line-no">1547</span><span id="line-1547"> public static Map<String, Map<String, Float>> getRegionDegreeLocalityMappingFromFS(</span> |
| <span class="source-line-no">1548</span><span id="line-1548"> final Configuration conf, final String desiredTable, int threadPoolSize) throws IOException {</span> |
| <span class="source-line-no">1549</span><span id="line-1549"> Map<String, Map<String, Float>> regionDegreeLocalityMapping = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">1550</span><span id="line-1550"> getRegionLocalityMappingFromFS(conf, desiredTable, threadPoolSize, regionDegreeLocalityMapping);</span> |
| <span class="source-line-no">1551</span><span id="line-1551"> return regionDegreeLocalityMapping;</span> |
| <span class="source-line-no">1552</span><span id="line-1552"> }</span> |
| <span class="source-line-no">1553</span><span id="line-1553"></span> |
| <span class="source-line-no">1554</span><span id="line-1554"> /**</span> |
| <span class="source-line-no">1555</span><span id="line-1555"> * This function is to scan the root path of the file system to get either the mapping between the</span> |
| <span class="source-line-no">1556</span><span id="line-1556"> * region name and its best locality region server or the degree of locality of each region on</span> |
| <span class="source-line-no">1557</span><span id="line-1557"> * each of the servers having at least one block of that region. The output map parameters are</span> |
| <span class="source-line-no">1558</span><span id="line-1558"> * both optional. the configuration to use the table you wish to scan locality for the thread pool</span> |
| <span class="source-line-no">1559</span><span id="line-1559"> * size to use the map into which to put the locality degree mapping or null, must be a</span> |
| <span class="source-line-no">1560</span><span id="line-1560"> * thread-safe implementation in case of file system errors or interrupts</span> |
| <span class="source-line-no">1561</span><span id="line-1561"> */</span> |
| <span class="source-line-no">1562</span><span id="line-1562"> private static void getRegionLocalityMappingFromFS(final Configuration conf,</span> |
| <span class="source-line-no">1563</span><span id="line-1563"> final String desiredTable, int threadPoolSize,</span> |
| <span class="source-line-no">1564</span><span id="line-1564"> final Map<String, Map<String, Float>> regionDegreeLocalityMapping) throws IOException {</span> |
| <span class="source-line-no">1565</span><span id="line-1565"> final FileSystem fs = FileSystem.get(conf);</span> |
| <span class="source-line-no">1566</span><span id="line-1566"> final Path rootPath = CommonFSUtils.getRootDir(conf);</span> |
| <span class="source-line-no">1567</span><span id="line-1567"> final long startTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1568</span><span id="line-1568"> final Path queryPath;</span> |
| <span class="source-line-no">1569</span><span id="line-1569"> // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*</span> |
| <span class="source-line-no">1570</span><span id="line-1570"> if (null == desiredTable) {</span> |
| <span class="source-line-no">1571</span><span id="line-1571"> queryPath =</span> |
| <span class="source-line-no">1572</span><span id="line-1572"> new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");</span> |
| <span class="source-line-no">1573</span><span id="line-1573"> } else {</span> |
| <span class="source-line-no">1574</span><span id="line-1574"> queryPath = new Path(</span> |
| <span class="source-line-no">1575</span><span id="line-1575"> CommonFSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");</span> |
| <span class="source-line-no">1576</span><span id="line-1576"> }</span> |
| <span class="source-line-no">1577</span><span id="line-1577"></span> |
| <span class="source-line-no">1578</span><span id="line-1578"> // reject all paths that are not appropriate</span> |
| <span class="source-line-no">1579</span><span id="line-1579"> PathFilter pathFilter = new PathFilter() {</span> |
| <span class="source-line-no">1580</span><span id="line-1580"> @Override</span> |
| <span class="source-line-no">1581</span><span id="line-1581"> public boolean accept(Path path) {</span> |
| <span class="source-line-no">1582</span><span id="line-1582"> // this is the region name; it may get some noise data</span> |
| <span class="source-line-no">1583</span><span id="line-1583"> if (null == path) {</span> |
| <span class="source-line-no">1584</span><span id="line-1584"> return false;</span> |
| <span class="source-line-no">1585</span><span id="line-1585"> }</span> |
| <span class="source-line-no">1586</span><span id="line-1586"></span> |
| <span class="source-line-no">1587</span><span id="line-1587"> // no parent?</span> |
| <span class="source-line-no">1588</span><span id="line-1588"> Path parent = path.getParent();</span> |
| <span class="source-line-no">1589</span><span id="line-1589"> if (null == parent) {</span> |
| <span class="source-line-no">1590</span><span id="line-1590"> return false;</span> |
| <span class="source-line-no">1591</span><span id="line-1591"> }</span> |
| <span class="source-line-no">1592</span><span id="line-1592"></span> |
| <span class="source-line-no">1593</span><span id="line-1593"> String regionName = path.getName();</span> |
| <span class="source-line-no">1594</span><span id="line-1594"> if (null == regionName) {</span> |
| <span class="source-line-no">1595</span><span id="line-1595"> return false;</span> |
| <span class="source-line-no">1596</span><span id="line-1596"> }</span> |
| <span class="source-line-no">1597</span><span id="line-1597"></span> |
| <span class="source-line-no">1598</span><span id="line-1598"> if (!regionName.toLowerCase(Locale.ROOT).matches("[0-9a-f]+")) {</span> |
| <span class="source-line-no">1599</span><span id="line-1599"> return false;</span> |
| <span class="source-line-no">1600</span><span id="line-1600"> }</span> |
| <span class="source-line-no">1601</span><span id="line-1601"> return true;</span> |
| <span class="source-line-no">1602</span><span id="line-1602"> }</span> |
| <span class="source-line-no">1603</span><span id="line-1603"> };</span> |
| <span class="source-line-no">1604</span><span id="line-1604"></span> |
| <span class="source-line-no">1605</span><span id="line-1605"> FileStatus[] statusList = fs.globStatus(queryPath, pathFilter);</span> |
| <span class="source-line-no">1606</span><span id="line-1606"></span> |
| <span class="source-line-no">1607</span><span id="line-1607"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">1608</span><span id="line-1608"> LOG.debug("Query Path: {} ; # list of files: {}", queryPath, Arrays.toString(statusList));</span> |
| <span class="source-line-no">1609</span><span id="line-1609"> }</span> |
| <span class="source-line-no">1610</span><span id="line-1610"></span> |
| <span class="source-line-no">1611</span><span id="line-1611"> if (null == statusList) {</span> |
| <span class="source-line-no">1612</span><span id="line-1612"> return;</span> |
| <span class="source-line-no">1613</span><span id="line-1613"> }</span> |
| <span class="source-line-no">1614</span><span id="line-1614"></span> |
| <span class="source-line-no">1615</span><span id="line-1615"> // lower the number of threads in case we have very few expected regions</span> |
| <span class="source-line-no">1616</span><span id="line-1616"> threadPoolSize = Math.min(threadPoolSize, statusList.length);</span> |
| <span class="source-line-no">1617</span><span id="line-1617"></span> |
| <span class="source-line-no">1618</span><span id="line-1618"> // run in multiple threads</span> |
| <span class="source-line-no">1619</span><span id="line-1619"> final ExecutorService tpe = Executors.newFixedThreadPool(threadPoolSize,</span> |
| <span class="source-line-no">1620</span><span id="line-1620"> new ThreadFactoryBuilder().setNameFormat("FSRegionQuery-pool-%d").setDaemon(true)</span> |
| <span class="source-line-no">1621</span><span id="line-1621"> .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());</span> |
| <span class="source-line-no">1622</span><span id="line-1622"> try {</span> |
| <span class="source-line-no">1623</span><span id="line-1623"> // ignore all file status items that are not of interest</span> |
| <span class="source-line-no">1624</span><span id="line-1624"> for (FileStatus regionStatus : statusList) {</span> |
| <span class="source-line-no">1625</span><span id="line-1625"> if (null == regionStatus || !regionStatus.isDirectory()) {</span> |
| <span class="source-line-no">1626</span><span id="line-1626"> continue;</span> |
| <span class="source-line-no">1627</span><span id="line-1627"> }</span> |
| <span class="source-line-no">1628</span><span id="line-1628"></span> |
| <span class="source-line-no">1629</span><span id="line-1629"> final Path regionPath = regionStatus.getPath();</span> |
| <span class="source-line-no">1630</span><span id="line-1630"> if (null != regionPath) {</span> |
| <span class="source-line-no">1631</span><span id="line-1631"> tpe.execute(new FSRegionScanner(fs, regionPath, null, regionDegreeLocalityMapping));</span> |
| <span class="source-line-no">1632</span><span id="line-1632"> }</span> |
| <span class="source-line-no">1633</span><span id="line-1633"> }</span> |
| <span class="source-line-no">1634</span><span id="line-1634"> } finally {</span> |
| <span class="source-line-no">1635</span><span id="line-1635"> tpe.shutdown();</span> |
| <span class="source-line-no">1636</span><span id="line-1636"> final long threadWakeFrequency = (long) conf.getInt(HConstants.THREAD_WAKE_FREQUENCY,</span> |
| <span class="source-line-no">1637</span><span id="line-1637"> HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);</span> |
| <span class="source-line-no">1638</span><span id="line-1638"> try {</span> |
| <span class="source-line-no">1639</span><span id="line-1639"> // here we wait until TPE terminates, which is either naturally or by</span> |
| <span class="source-line-no">1640</span><span id="line-1640"> // exceptions in the execution of the threads</span> |
| <span class="source-line-no">1641</span><span id="line-1641"> while (!tpe.awaitTermination(threadWakeFrequency, TimeUnit.MILLISECONDS)) {</span> |
| <span class="source-line-no">1642</span><span id="line-1642"> // printing out rough estimate, so as to not introduce</span> |
| <span class="source-line-no">1643</span><span id="line-1643"> // AtomicInteger</span> |
| <span class="source-line-no">1644</span><span id="line-1644"> LOG.info("Locality checking is underway: { Scanned Regions : "</span> |
| <span class="source-line-no">1645</span><span id="line-1645"> + ((ThreadPoolExecutor) tpe).getCompletedTaskCount() + "/"</span> |
| <span class="source-line-no">1646</span><span id="line-1646"> + ((ThreadPoolExecutor) tpe).getTaskCount() + " }");</span> |
| <span class="source-line-no">1647</span><span id="line-1647"> }</span> |
| <span class="source-line-no">1648</span><span id="line-1648"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">1649</span><span id="line-1649"> Thread.currentThread().interrupt();</span> |
| <span class="source-line-no">1650</span><span id="line-1650"> throw (InterruptedIOException) new InterruptedIOException().initCause(e);</span> |
| <span class="source-line-no">1651</span><span id="line-1651"> }</span> |
| <span class="source-line-no">1652</span><span id="line-1652"> }</span> |
| <span class="source-line-no">1653</span><span id="line-1653"></span> |
| <span class="source-line-no">1654</span><span id="line-1654"> long overhead = EnvironmentEdgeManager.currentTime() - startTime;</span> |
| <span class="source-line-no">1655</span><span id="line-1655"> LOG.info("Scan DFS for locality info takes {}ms", overhead);</span> |
| <span class="source-line-no">1656</span><span id="line-1656"> }</span> |
| <span class="source-line-no">1657</span><span id="line-1657"></span> |
| <span class="source-line-no">1658</span><span id="line-1658"> /**</span> |
| <span class="source-line-no">1659</span><span id="line-1659"> * Do our short circuit read setup. Checks buffer size to use and whether to do checksumming in</span> |
| <span class="source-line-no">1660</span><span id="line-1660"> * hbase or hdfs.</span> |
| <span class="source-line-no">1661</span><span id="line-1661"> */</span> |
| <span class="source-line-no">1662</span><span id="line-1662"> public static void setupShortCircuitRead(final Configuration conf) {</span> |
| <span class="source-line-no">1663</span><span id="line-1663"> // Check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property.</span> |
| <span class="source-line-no">1664</span><span id="line-1664"> boolean shortCircuitSkipChecksum =</span> |
| <span class="source-line-no">1665</span><span id="line-1665"> conf.getBoolean("dfs.client.read.shortcircuit.skip.checksum", false);</span> |
| <span class="source-line-no">1666</span><span id="line-1666"> boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);</span> |
| <span class="source-line-no">1667</span><span id="line-1667"> if (shortCircuitSkipChecksum) {</span> |
| <span class="source-line-no">1668</span><span id="line-1668"> LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not "</span> |
| <span class="source-line-no">1669</span><span id="line-1669"> + "be set to true."</span> |
| <span class="source-line-no">1670</span><span id="line-1670"> + (useHBaseChecksum</span> |
| <span class="source-line-no">1671</span><span id="line-1671"> ? " HBase checksum doesn't require "</span> |
| <span class="source-line-no">1672</span><span id="line-1672"> + "it, see https://issues.apache.org/jira/browse/HBASE-6868."</span> |
| <span class="source-line-no">1673</span><span id="line-1673"> : ""));</span> |
| <span class="source-line-no">1674</span><span id="line-1674"> assert !shortCircuitSkipChecksum; // this will fail if assertions are on</span> |
| <span class="source-line-no">1675</span><span id="line-1675"> }</span> |
| <span class="source-line-no">1676</span><span id="line-1676"> checkShortCircuitReadBufferSize(conf);</span> |
| <span class="source-line-no">1677</span><span id="line-1677"> }</span> |
| <span class="source-line-no">1678</span><span id="line-1678"></span> |
| <span class="source-line-no">1679</span><span id="line-1679"> /**</span> |
| <span class="source-line-no">1680</span><span id="line-1680"> * Check if short circuit read buffer size is set and if not, set it to hbase value.</span> |
| <span class="source-line-no">1681</span><span id="line-1681"> */</span> |
| <span class="source-line-no">1682</span><span id="line-1682"> public static void checkShortCircuitReadBufferSize(final Configuration conf) {</span> |
| <span class="source-line-no">1683</span><span id="line-1683"> final int defaultSize = HConstants.DEFAULT_BLOCKSIZE * 2;</span> |
| <span class="source-line-no">1684</span><span id="line-1684"> final int notSet = -1;</span> |
| <span class="source-line-no">1685</span><span id="line-1685"> // DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY is only defined in h2</span> |
| <span class="source-line-no">1686</span><span id="line-1686"> final String dfsKey = "dfs.client.read.shortcircuit.buffer.size";</span> |
| <span class="source-line-no">1687</span><span id="line-1687"> int size = conf.getInt(dfsKey, notSet);</span> |
| <span class="source-line-no">1688</span><span id="line-1688"> // If a size is set, return -- we will use it.</span> |
| <span class="source-line-no">1689</span><span id="line-1689"> if (size != notSet) return;</span> |
| <span class="source-line-no">1690</span><span id="line-1690"> // But short circuit buffer size is normally not set. Put in place the hbase wanted size.</span> |
| <span class="source-line-no">1691</span><span id="line-1691"> int hbaseSize = conf.getInt("hbase." + dfsKey, defaultSize);</span> |
| <span class="source-line-no">1692</span><span id="line-1692"> conf.setIfUnset(dfsKey, Integer.toString(hbaseSize));</span> |
| <span class="source-line-no">1693</span><span id="line-1693"> }</span> |
| <span class="source-line-no">1694</span><span id="line-1694"></span> |
| <span class="source-line-no">1695</span><span id="line-1695"> /**</span> |
| <span class="source-line-no">1696</span><span id="line-1696"> * Returns The DFSClient DFSHedgedReadMetrics instance or null if can't be found or not on hdfs.</span> |
| <span class="source-line-no">1697</span><span id="line-1697"> */</span> |
| <span class="source-line-no">1698</span><span id="line-1698"> public static DFSHedgedReadMetrics getDFSHedgedReadMetrics(final Configuration c)</span> |
| <span class="source-line-no">1699</span><span id="line-1699"> throws IOException {</span> |
| <span class="source-line-no">1700</span><span id="line-1700"> if (!CommonFSUtils.isHDFS(c)) {</span> |
| <span class="source-line-no">1701</span><span id="line-1701"> return null;</span> |
| <span class="source-line-no">1702</span><span id="line-1702"> }</span> |
| <span class="source-line-no">1703</span><span id="line-1703"> // getHedgedReadMetrics is package private. Get the DFSClient instance that is internal</span> |
| <span class="source-line-no">1704</span><span id="line-1704"> // to the DFS FS instance and make the method getHedgedReadMetrics accessible, then invoke it</span> |
| <span class="source-line-no">1705</span><span id="line-1705"> // to get the singleton instance of DFSHedgedReadMetrics shared by DFSClients.</span> |
| <span class="source-line-no">1706</span><span id="line-1706"> final String name = "getHedgedReadMetrics";</span> |
| <span class="source-line-no">1707</span><span id="line-1707"> DFSClient dfsclient = ((DistributedFileSystem) FileSystem.get(c)).getClient();</span> |
| <span class="source-line-no">1708</span><span id="line-1708"> Method m;</span> |
| <span class="source-line-no">1709</span><span id="line-1709"> try {</span> |
| <span class="source-line-no">1710</span><span id="line-1710"> m = dfsclient.getClass().getDeclaredMethod(name);</span> |
| <span class="source-line-no">1711</span><span id="line-1711"> } catch (NoSuchMethodException e) {</span> |
| <span class="source-line-no">1712</span><span id="line-1712"> LOG.warn(</span> |
| <span class="source-line-no">1713</span><span id="line-1713"> "Failed find method " + name + " in dfsclient; no hedged read metrics: " + e.getMessage());</span> |
| <span class="source-line-no">1714</span><span id="line-1714"> return null;</span> |
| <span class="source-line-no">1715</span><span id="line-1715"> } catch (SecurityException e) {</span> |
| <span class="source-line-no">1716</span><span id="line-1716"> LOG.warn(</span> |
| <span class="source-line-no">1717</span><span id="line-1717"> "Failed find method " + name + " in dfsclient; no hedged read metrics: " + e.getMessage());</span> |
| <span class="source-line-no">1718</span><span id="line-1718"> return null;</span> |
| <span class="source-line-no">1719</span><span id="line-1719"> }</span> |
| <span class="source-line-no">1720</span><span id="line-1720"> m.setAccessible(true);</span> |
| <span class="source-line-no">1721</span><span id="line-1721"> try {</span> |
| <span class="source-line-no">1722</span><span id="line-1722"> return (DFSHedgedReadMetrics) m.invoke(dfsclient);</span> |
| <span class="source-line-no">1723</span><span id="line-1723"> } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {</span> |
| <span class="source-line-no">1724</span><span id="line-1724"> LOG.warn("Failed invoking method " + name + " on dfsclient; no hedged read metrics: "</span> |
| <span class="source-line-no">1725</span><span id="line-1725"> + e.getMessage());</span> |
| <span class="source-line-no">1726</span><span id="line-1726"> return null;</span> |
| <span class="source-line-no">1727</span><span id="line-1727"> }</span> |
| <span class="source-line-no">1728</span><span id="line-1728"> }</span> |
| <span class="source-line-no">1729</span><span id="line-1729"></span> |
| <span class="source-line-no">1730</span><span id="line-1730"> public static List<Path> copyFilesParallel(FileSystem srcFS, Path src, FileSystem dstFS, Path dst,</span> |
| <span class="source-line-no">1731</span><span id="line-1731"> Configuration conf, int threads) throws IOException {</span> |
| <span class="source-line-no">1732</span><span id="line-1732"> ExecutorService pool = Executors.newFixedThreadPool(threads);</span> |
| <span class="source-line-no">1733</span><span id="line-1733"> List<Future<Void>> futures = new ArrayList<>();</span> |
| <span class="source-line-no">1734</span><span id="line-1734"> List<Path> traversedPaths;</span> |
| <span class="source-line-no">1735</span><span id="line-1735"> try {</span> |
| <span class="source-line-no">1736</span><span id="line-1736"> traversedPaths = copyFiles(srcFS, src, dstFS, dst, conf, pool, futures);</span> |
| <span class="source-line-no">1737</span><span id="line-1737"> for (Future<Void> future : futures) {</span> |
| <span class="source-line-no">1738</span><span id="line-1738"> future.get();</span> |
| <span class="source-line-no">1739</span><span id="line-1739"> }</span> |
| <span class="source-line-no">1740</span><span id="line-1740"> } catch (ExecutionException | InterruptedException | IOException e) {</span> |
| <span class="source-line-no">1741</span><span id="line-1741"> throw new IOException("Copy snapshot reference files failed", e);</span> |
| <span class="source-line-no">1742</span><span id="line-1742"> } finally {</span> |
| <span class="source-line-no">1743</span><span id="line-1743"> pool.shutdownNow();</span> |
| <span class="source-line-no">1744</span><span id="line-1744"> }</span> |
| <span class="source-line-no">1745</span><span id="line-1745"> return traversedPaths;</span> |
| <span class="source-line-no">1746</span><span id="line-1746"> }</span> |
| <span class="source-line-no">1747</span><span id="line-1747"></span> |
| <span class="source-line-no">1748</span><span id="line-1748"> private static List<Path> copyFiles(FileSystem srcFS, Path src, FileSystem dstFS, Path dst,</span> |
| <span class="source-line-no">1749</span><span id="line-1749"> Configuration conf, ExecutorService pool, List<Future<Void>> futures) throws IOException {</span> |
| <span class="source-line-no">1750</span><span id="line-1750"> List<Path> traversedPaths = new ArrayList<>();</span> |
| <span class="source-line-no">1751</span><span id="line-1751"> traversedPaths.add(dst);</span> |
| <span class="source-line-no">1752</span><span id="line-1752"> FileStatus currentFileStatus = srcFS.getFileStatus(src);</span> |
| <span class="source-line-no">1753</span><span id="line-1753"> if (currentFileStatus.isDirectory()) {</span> |
| <span class="source-line-no">1754</span><span id="line-1754"> if (!dstFS.mkdirs(dst)) {</span> |
| <span class="source-line-no">1755</span><span id="line-1755"> throw new IOException("Create directory failed: " + dst);</span> |
| <span class="source-line-no">1756</span><span id="line-1756"> }</span> |
| <span class="source-line-no">1757</span><span id="line-1757"> FileStatus[] subPaths = srcFS.listStatus(src);</span> |
| <span class="source-line-no">1758</span><span id="line-1758"> for (FileStatus subPath : subPaths) {</span> |
| <span class="source-line-no">1759</span><span id="line-1759"> traversedPaths.addAll(copyFiles(srcFS, subPath.getPath(), dstFS,</span> |
| <span class="source-line-no">1760</span><span id="line-1760"> new Path(dst, subPath.getPath().getName()), conf, pool, futures));</span> |
| <span class="source-line-no">1761</span><span id="line-1761"> }</span> |
| <span class="source-line-no">1762</span><span id="line-1762"> } else {</span> |
| <span class="source-line-no">1763</span><span id="line-1763"> Future<Void> future = pool.submit(() -> {</span> |
| <span class="source-line-no">1764</span><span id="line-1764"> FileUtil.copy(srcFS, src, dstFS, dst, false, false, conf);</span> |
| <span class="source-line-no">1765</span><span id="line-1765"> return null;</span> |
| <span class="source-line-no">1766</span><span id="line-1766"> });</span> |
| <span class="source-line-no">1767</span><span id="line-1767"> futures.add(future);</span> |
| <span class="source-line-no">1768</span><span id="line-1768"> }</span> |
| <span class="source-line-no">1769</span><span id="line-1769"> return traversedPaths;</span> |
| <span class="source-line-no">1770</span><span id="line-1770"> }</span> |
| <span class="source-line-no">1771</span><span id="line-1771"></span> |
| <span class="source-line-no">1772</span><span id="line-1772"> /** Returns A set containing all namenode addresses of fs */</span> |
| <span class="source-line-no">1773</span><span id="line-1773"> private static Set<InetSocketAddress> getNNAddresses(DistributedFileSystem fs,</span> |
| <span class="source-line-no">1774</span><span id="line-1774"> Configuration conf) {</span> |
| <span class="source-line-no">1775</span><span id="line-1775"> Set<InetSocketAddress> addresses = new HashSet<>();</span> |
| <span class="source-line-no">1776</span><span id="line-1776"> String serviceName = fs.getCanonicalServiceName();</span> |
| <span class="source-line-no">1777</span><span id="line-1777"></span> |
| <span class="source-line-no">1778</span><span id="line-1778"> if (serviceName.startsWith("ha-hdfs")) {</span> |
| <span class="source-line-no">1779</span><span id="line-1779"> try {</span> |
| <span class="source-line-no">1780</span><span id="line-1780"> Map<String, Map<String, InetSocketAddress>> addressMap =</span> |
| <span class="source-line-no">1781</span><span id="line-1781"> DFSUtil.getNNServiceRpcAddressesForCluster(conf);</span> |
| <span class="source-line-no">1782</span><span id="line-1782"> String nameService = serviceName.substring(serviceName.indexOf(":") + 1);</span> |
| <span class="source-line-no">1783</span><span id="line-1783"> if (addressMap.containsKey(nameService)) {</span> |
| <span class="source-line-no">1784</span><span id="line-1784"> Map<String, InetSocketAddress> nnMap = addressMap.get(nameService);</span> |
| <span class="source-line-no">1785</span><span id="line-1785"> for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {</span> |
| <span class="source-line-no">1786</span><span id="line-1786"> InetSocketAddress addr = e2.getValue();</span> |
| <span class="source-line-no">1787</span><span id="line-1787"> addresses.add(addr);</span> |
| <span class="source-line-no">1788</span><span id="line-1788"> }</span> |
| <span class="source-line-no">1789</span><span id="line-1789"> }</span> |
| <span class="source-line-no">1790</span><span id="line-1790"> } catch (Exception e) {</span> |
| <span class="source-line-no">1791</span><span id="line-1791"> LOG.warn("DFSUtil.getNNServiceRpcAddresses failed. serviceName=" + serviceName, e);</span> |
| <span class="source-line-no">1792</span><span id="line-1792"> }</span> |
| <span class="source-line-no">1793</span><span id="line-1793"> } else {</span> |
| <span class="source-line-no">1794</span><span id="line-1794"> URI uri = fs.getUri();</span> |
| <span class="source-line-no">1795</span><span id="line-1795"> int port = uri.getPort();</span> |
| <span class="source-line-no">1796</span><span id="line-1796"> if (port < 0) {</span> |
| <span class="source-line-no">1797</span><span id="line-1797"> int idx = serviceName.indexOf(':');</span> |
| <span class="source-line-no">1798</span><span id="line-1798"> port = Integer.parseInt(serviceName.substring(idx + 1));</span> |
| <span class="source-line-no">1799</span><span id="line-1799"> }</span> |
| <span class="source-line-no">1800</span><span id="line-1800"> InetSocketAddress addr = new InetSocketAddress(uri.getHost(), port);</span> |
| <span class="source-line-no">1801</span><span id="line-1801"> addresses.add(addr);</span> |
| <span class="source-line-no">1802</span><span id="line-1802"> }</span> |
| <span class="source-line-no">1803</span><span id="line-1803"></span> |
| <span class="source-line-no">1804</span><span id="line-1804"> return addresses;</span> |
| <span class="source-line-no">1805</span><span id="line-1805"> }</span> |
| <span class="source-line-no">1806</span><span id="line-1806"></span> |
| <span class="source-line-no">1807</span><span id="line-1807"> /**</span> |
| <span class="source-line-no">1808</span><span id="line-1808"> * @param conf the Configuration of HBase</span> |
| <span class="source-line-no">1809</span><span id="line-1809"> * @return Whether srcFs and desFs are on same hdfs or not</span> |
| <span class="source-line-no">1810</span><span id="line-1810"> */</span> |
| <span class="source-line-no">1811</span><span id="line-1811"> public static boolean isSameHdfs(Configuration conf, FileSystem srcFs, FileSystem desFs) {</span> |
| <span class="source-line-no">1812</span><span id="line-1812"> // By getCanonicalServiceName, we could make sure both srcFs and desFs</span> |
| <span class="source-line-no">1813</span><span id="line-1813"> // show a unified format which contains scheme, host and port.</span> |
| <span class="source-line-no">1814</span><span id="line-1814"> String srcServiceName = srcFs.getCanonicalServiceName();</span> |
| <span class="source-line-no">1815</span><span id="line-1815"> String desServiceName = desFs.getCanonicalServiceName();</span> |
| <span class="source-line-no">1816</span><span id="line-1816"></span> |
| <span class="source-line-no">1817</span><span id="line-1817"> if (srcServiceName == null || desServiceName == null) {</span> |
| <span class="source-line-no">1818</span><span id="line-1818"> return false;</span> |
| <span class="source-line-no">1819</span><span id="line-1819"> }</span> |
| <span class="source-line-no">1820</span><span id="line-1820"> if (srcServiceName.equals(desServiceName)) {</span> |
| <span class="source-line-no">1821</span><span id="line-1821"> return true;</span> |
| <span class="source-line-no">1822</span><span id="line-1822"> }</span> |
| <span class="source-line-no">1823</span><span id="line-1823"> if (srcServiceName.startsWith("ha-hdfs") && desServiceName.startsWith("ha-hdfs")) {</span> |
| <span class="source-line-no">1824</span><span id="line-1824"> Collection<String> internalNameServices =</span> |
| <span class="source-line-no">1825</span><span id="line-1825"> conf.getTrimmedStringCollection("dfs.internal.nameservices");</span> |
| <span class="source-line-no">1826</span><span id="line-1826"> if (!internalNameServices.isEmpty()) {</span> |
| <span class="source-line-no">1827</span><span id="line-1827"> if (internalNameServices.contains(srcServiceName.split(":")[1])) {</span> |
| <span class="source-line-no">1828</span><span id="line-1828"> return true;</span> |
| <span class="source-line-no">1829</span><span id="line-1829"> } else {</span> |
| <span class="source-line-no">1830</span><span id="line-1830"> return false;</span> |
| <span class="source-line-no">1831</span><span id="line-1831"> }</span> |
| <span class="source-line-no">1832</span><span id="line-1832"> }</span> |
| <span class="source-line-no">1833</span><span id="line-1833"> }</span> |
| <span class="source-line-no">1834</span><span id="line-1834"> if (srcFs instanceof DistributedFileSystem && desFs instanceof DistributedFileSystem) {</span> |
| <span class="source-line-no">1835</span><span id="line-1835"> // If one serviceName is an HA format while the other is a non-HA format,</span> |
| <span class="source-line-no">1836</span><span id="line-1836"> // maybe they refer to the same FileSystem.</span> |
| <span class="source-line-no">1837</span><span id="line-1837"> // For example, srcFs is "ha-hdfs://nameservices" and desFs is "hdfs://activeNamenode:port"</span> |
| <span class="source-line-no">1838</span><span id="line-1838"> Set<InetSocketAddress> srcAddrs = getNNAddresses((DistributedFileSystem) srcFs, conf);</span> |
| <span class="source-line-no">1839</span><span id="line-1839"> Set<InetSocketAddress> desAddrs = getNNAddresses((DistributedFileSystem) desFs, conf);</span> |
| <span class="source-line-no">1840</span><span id="line-1840"> if (Sets.intersection(srcAddrs, desAddrs).size() > 0) {</span> |
| <span class="source-line-no">1841</span><span id="line-1841"> return true;</span> |
| <span class="source-line-no">1842</span><span id="line-1842"> }</span> |
| <span class="source-line-no">1843</span><span id="line-1843"> }</span> |
| <span class="source-line-no">1844</span><span id="line-1844"></span> |
| <span class="source-line-no">1845</span><span id="line-1845"> return false;</span> |
| <span class="source-line-no">1846</span><span id="line-1846"> }</span> |
| <span class="source-line-no">1847</span><span id="line-1847">}</span> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </pre> |
| </div> |
| </main> |
| </body> |
| </html> |
